From 4b156c380053e17dd861a329cba0902cec6c754a Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Mon, 5 Nov 2001 19:34:25 +0000 Subject: [PATCH] bkg-daemon-dont-break-64bit-pointers-and-handle-requests-in-order-20011102 This patch fixes a problem with 64-bit pointers being munged by the background daemons (by separating sizes and pointers into separate variables -- this bug was apparently introduced by the 64-bit file support patch), and makes the background daemons handle requests in order they came in. The latter will be mostly just useful for some prefetching and fine grained dcache-locking patches --- src/afs/IRIX/osi_vnodeops.c | 4 +- src/afs/VNOPS/afs_vnop_read.c | 11 ++-- src/afs/VNOPS/afs_vnop_write.c | 2 +- src/afs/afs.h | 6 +- src/afs/afs_daemons.c | 103 +++++++++++++++++++-------------- src/afs/afs_pioctl.c | 3 +- src/venus/kdump.c | 2 +- 7 files changed, 73 insertions(+), 58 deletions(-) diff --git a/src/afs/IRIX/osi_vnodeops.c b/src/afs/IRIX/osi_vnodeops.c index ef5fbdb800..809d991ee1 100644 --- a/src/afs/IRIX/osi_vnodeops.c +++ b/src/afs/IRIX/osi_vnodeops.c @@ -967,8 +967,8 @@ static int afs_delmap(OSI_VC_ARG(avc), off, prp, addr, len, prot, maxprot, AFS_RWUNLOCK(vp, VRWLOCK_WRITE); /* at least one daemon is idle, so ask it to do the store. * Also, note that we don't lock it any more... */ - tb = afs_BQueue(BOP_STORE, avc, 0, 1, acred, (long)acred->cr_uid, - 0L, 0L, 0L); + tb = afs_BQueue(BOP_STORE, avc, 0, 1, acred, + (afs_size_t) acred->cr_uid, 0L, (void *) 0); /* sleep waiting for the store to start, then retrieve error code */ while ((tb->flags & BUVALID) == 0) { tb->flags |= BUWAIT; diff --git a/src/afs/VNOPS/afs_vnop_read.c b/src/afs/VNOPS/afs_vnop_read.c index c93edd6d03..8d337e854a 100644 --- a/src/afs/VNOPS/afs_vnop_read.c +++ b/src/afs/VNOPS/afs_vnop_read.c @@ -165,8 +165,8 @@ tagain: /* start the daemon (may already be running, however) */ tdc->flags |= DFFetchReq; bp = afs_BQueue(BOP_FETCH, avc, B_DONTWAIT, 0, acred, - (afs_size_t)filePos, (afs_size_t) tdc, - (afs_size_t) 0, (afs_size_t) 0); + (afs_size_t)filePos, (afs_size_t) 0, + tdc); if (!bp) { tdc->flags &= ~DFFetchReq; trybusy = 0; /* Avoid bkg daemon since they're too busy */ @@ -318,8 +318,7 @@ void afs_PrefetchChunk(struct vcache *avc, struct dcache *adc, mutex_exit(&tdc->lock); #endif bp = afs_BQueue(BOP_FETCH, avc, B_DONTWAIT, 0, acred, - (afs_size_t) offset, (afs_size_t) tdc, - (afs_size_t) 1, (afs_size_t) 0); + (afs_size_t) offset, (afs_size_t) 1, tdc); if (!bp) { /* Bkg table full; just abort non-important prefetching to avoid deadlocks */ tdc->flags &= ~(DFNextStarted | DFFetchReq); @@ -614,8 +613,8 @@ tagain: munlocked = 1; #endif bp = afs_BQueue(BOP_FETCH, avc, B_DONTWAIT, 0, acred, - (afs_size_t)filePos, (afs_size_t) tdc, - (afs_size_t) 0, (afs_size_t) 0); + (afs_size_t)filePos, (afs_size_t) 0, + tdc); if (!bp) { /* Bkg table full; retry deadlocks */ tdc->flags &= ~DFFetchReq; diff --git a/src/afs/VNOPS/afs_vnop_write.c b/src/afs/VNOPS/afs_vnop_write.c index 1e1aa79ad2..62d7e5d5e4 100644 --- a/src/afs/VNOPS/afs_vnop_write.c +++ b/src/afs/VNOPS/afs_vnop_write.c @@ -844,7 +844,7 @@ afs_close(OSI_VC_ARG(avc), aflags, acred) Also, note that we don't lock it any more... */ tb = afs_BQueue(BOP_STORE, avc, 0, 1, acred, (afs_size_t) acred->cr_uid, (afs_size_t) 0, - (afs_size_t) 0, (afs_size_t) 0); + (void *) 0); /* sleep waiting for the store to start, then retrieve error code */ while ((tb->flags & BUVALID) == 0) { tb->flags |= BUWAIT; diff --git a/src/afs/afs.h b/src/afs/afs.h index bd6bf5b4b2..86a999688a 100644 --- a/src/afs/afs.h +++ b/src/afs/afs.h @@ -116,11 +116,13 @@ struct sysname_info { struct brequest { struct vcache *vnode; /* vnode to use, with vrefcount bumped */ struct AFS_UCRED *cred; /* credentials to use for operation */ - afs_size_t parm[BPARMS]; /* random parameters */ - afs_int32 code; /* return code */ + afs_size_t size_parm[BPARMS]; /* random parameters */ + void *ptr_parm[BPARMS]; /* pointer parameters */ + afs_int32 code; /* return code */ short refCount; /* use counter for this structure */ char opcode; /* what to do (store, fetch, etc) */ char flags; /* free, etc */ + afs_int32 ts; /* counter "timestamp" */ }; struct SecretToken { diff --git a/src/afs/afs_daemons.c b/src/afs/afs_daemons.c index 16c5fe6270..00e91c0cd8 100644 --- a/src/afs/afs_daemons.c +++ b/src/afs/afs_daemons.c @@ -28,6 +28,7 @@ short afs_brsWaiters = 0; /* number of users waiting for brs buffers */ short afs_brsDaemons = 0; /* number of daemons waiting for brs requests */ struct brequest afs_brs[NBRS]; /* request structures */ struct afs_osi_WaitHandle AFS_WaitHandler, AFS_CSWaitHandler; +static int afs_brs_count = 0; /* request counter, to service reqs in order */ static int rxepoch_checked=0; #define afs_CheckRXEpoch() {if (rxepoch_checked == 0 && rxkad_EpochWasSet) { \ @@ -346,7 +347,7 @@ afs_CheckRootVolume () { else return ENOENT; } -/* parm 0 is the pathname, parm 1 to the fetch is the chunk number */ +/* ptr_parm 0 is the pathname, size_parm 0 to the fetch is the chunk number */ void BPath(ab) register struct brequest *ab; { register struct dcache *tdc; @@ -363,14 +364,14 @@ void BPath(ab) if (code = afs_InitReq(&treq, ab->cred)) return; AFS_GUNLOCK(); #ifdef AFS_LINUX22_ENV - code = gop_lookupname((char *)ab->parm[0], AFS_UIOSYS, 1, (struct vnode **) 0, &dp); + code = gop_lookupname((char *)ab->ptr_parm[0], AFS_UIOSYS, 1, (struct vnode **) 0, &dp); if (dp) tvn = (struct vnode*)dp->d_inode; #else - code = gop_lookupname((char *)ab->parm[0], AFS_UIOSYS, 1, (struct vnode **) 0, (struct vnode **)&tvn); + code = gop_lookupname((char *)ab->ptr_parm[0], AFS_UIOSYS, 1, (struct vnode **) 0, (struct vnode **)&tvn); #endif AFS_GLOCK(); - osi_FreeLargeSpace((char *)ab->parm[0]); /* free path name buffer here */ + osi_FreeLargeSpace((char *)ab->ptr_parm[0]); /* free path name buffer here */ if (code) return; /* now path may not have been in afs, so check that before calling our cache manager */ if (!tvn || !IsAfsVnode((struct vnode *) tvn)) { @@ -394,7 +395,7 @@ void BPath(ab) tvc = (struct vcache *) tvn; #endif /* here we know its an afs vnode, so we can get the data for the chunk */ - tdc = afs_GetDCache(tvc, (afs_size_t) ab->parm[1], &treq, &offset, &len, 1); + tdc = afs_GetDCache(tvc, ab->size_parm[0], &treq, &offset, &len, 1); if (tdc) { afs_PutDCache(tdc); } @@ -409,8 +410,9 @@ void BPath(ab) #endif } -/* parm 0 to the fetch is the chunk number; parm 1 is the dcache entry to wakeup, - * parm 2 is true iff we should release the dcache entry here. +/* size_parm 0 to the fetch is the chunk number, + * ptr_parm 0 is the dcache entry to wakeup, + * size_parm 1 is true iff we should release the dcache entry here. */ void BPrefetch(ab) register struct brequest *ab; { @@ -422,7 +424,7 @@ void BPrefetch(ab) AFS_STATCNT(BPrefetch); if (len = afs_InitReq(&treq, ab->cred)) return; tvc = ab->vnode; - tdc = afs_GetDCache(tvc, (afs_size_t)ab->parm[0], &treq, &offset, &len, 1); + tdc = afs_GetDCache(tvc, ab->size_parm[0], &treq, &offset, &len, 1); if (tdc) { afs_PutDCache(tdc); } @@ -430,10 +432,10 @@ void BPrefetch(ab) * use tdc from GetDCache since afs_GetDCache may fail, but someone may * be waiting for our wakeup anyway. */ - tdc = (struct dcache *) (ab->parm[1]); + tdc = (struct dcache *) (ab->ptr_parm[0]); tdc->flags &= ~DFFetchReq; afs_osi_Wakeup(&tdc->validPos); - if ((afs_size_t)ab->parm[2]) { + if (ab->size_parm[1]) { #ifdef AFS_SUN5_ENVX mutex_enter(&tdc->lock); tdc->refCount--; @@ -512,13 +514,13 @@ int afs_BBusy() { return 1; } -struct brequest *afs_BQueue(aopcode, avc, dontwait, ause, acred, aparm0, aparm1, aparm2, aparm3) +struct brequest *afs_BQueue(aopcode, avc, dontwait, ause, acred, asparm0, asparm1, apparm0) register short aopcode; afs_int32 ause, dontwait; register struct vcache *avc; struct AFS_UCRED *acred; - /* On 64 bit platforms, "long" does the right thing. */ - afs_size_t aparm0, aparm1, aparm2, aparm3; + afs_size_t asparm0, asparm1; + void *apparm0; { register int i; register struct brequest *tb; @@ -544,12 +546,12 @@ struct brequest *afs_BQueue(aopcode, avc, dontwait, ause, acred, aparm0, aparm1, #endif } tb->refCount = ause+1; - tb->parm[0] = aparm0; - tb->parm[1] = aparm1; - tb->parm[2] = aparm2; - tb->parm[3] = aparm3; + tb->size_parm[0] = asparm0; + tb->size_parm[1] = asparm1; + tb->ptr_parm[0] = apparm0; tb->flags = 0; tb->code = 0; + tb->ts = afs_brs_count++; /* if daemons are waiting for work, wake them up */ if (afs_brsDaemons > 0) { afs_osi_Wakeup(&afs_brsDaemons); @@ -1231,6 +1233,9 @@ void afs_BackgroundDaemon() { MObtainWriteLock(&afs_xbrs,302); while (1) { + int min_ts; + struct brequest *min_tb; + if (afs_termState == AFSOP_STOP_BKG) { if (--afs_nbrs <= 0) afs_termState = AFSOP_STOP_TRUNCDAEMON; @@ -1238,42 +1243,50 @@ void afs_BackgroundDaemon() { afs_osi_Wakeup(&afs_termState); return; } - + /* find a request */ tb = afs_brs; foundAny = 0; - for(i=0;irefCount > 0) && !(tb->flags & BSTARTED)) { /* new request, not yet picked up */ - tb->flags |= BSTARTED; - MReleaseWriteLock(&afs_xbrs); - foundAny = 1; - afs_Trace1(afs_iclSetp, CM_TRACE_BKG1, - ICL_TYPE_INT32, tb->opcode); - if (tb->opcode == BOP_FETCH) - BPrefetch(tb); - else if (tb->opcode == BOP_STORE) - BStore(tb); - else if (tb->opcode == BOP_PATH) - BPath(tb); - else panic("background bop"); - if (tb->vnode) { -#ifdef AFS_DEC_ENV - tb->vnode->vrefCount--; /* fix up reference count */ -#else - AFS_RELE((struct vnode *)(tb->vnode)); /* MUST call vnode layer or could lose vnodes */ -#endif - tb->vnode = (struct vcache *) 0; + if ((min_tb && (min_ts - tb->ts > 0)) || !min_tb) { + min_tb = tb; + min_ts = tb->ts; } - if (tb->cred) { - crfree(tb->cred); - tb->cred = (struct AFS_UCRED *) 0; - } - afs_BRelease(tb); /* this grabs and releases afs_xbrs lock */ - MObtainWriteLock(&afs_xbrs,305); } } + if (tb = min_tb) { + /* claim and process this request */ + tb->flags |= BSTARTED; + MReleaseWriteLock(&afs_xbrs); + foundAny = 1; + afs_Trace1(afs_iclSetp, CM_TRACE_BKG1, + ICL_TYPE_INT32, tb->opcode); + if (tb->opcode == BOP_FETCH) + BPrefetch(tb); + else if (tb->opcode == BOP_STORE) + BStore(tb); + else if (tb->opcode == BOP_PATH) + BPath(tb); + else panic("background bop"); + if (tb->vnode) { +#ifdef AFS_DEC_ENV + tb->vnode->vrefCount--; /* fix up reference count */ +#else + AFS_RELE((struct vnode *)(tb->vnode)); /* MUST call vnode layer or could lose vnodes */ +#endif + tb->vnode = (struct vcache *) 0; + } + if (tb->cred) { + crfree(tb->cred); + tb->cred = (struct AFS_UCRED *) 0; + } + afs_BRelease(tb); /* this grabs and releases afs_xbrs lock */ + MObtainWriteLock(&afs_xbrs,305); + } if (!foundAny) { /* wait for new request */ afs_brsDaemons++; diff --git a/src/afs/afs_pioctl.c b/src/afs/afs_pioctl.c index bfc89a9e2f..ae55c75b86 100644 --- a/src/afs/afs_pioctl.c +++ b/src/afs/afs_pioctl.c @@ -2079,7 +2079,8 @@ struct AFS_UCRED *acred; osi_FreeLargeSpace(tp); return EWOULDBLOCK; /* pretty close */ } - afs_BQueue(BOP_PATH, (struct vcache*)0, 0, 0, acred, (long)tp, 0L, 0L, 0L); + afs_BQueue(BOP_PATH, (struct vcache*)0, 0, 0, acred, + (afs_size_t) 0, (afs_size_t) 0, tp); return 0; } diff --git a/src/venus/kdump.c b/src/venus/kdump.c index e9493a77fc..bda714f1c7 100644 --- a/src/venus/kdump.c +++ b/src/venus/kdump.c @@ -2682,7 +2682,7 @@ void print_bkg(kmem) if (uentry->refCount == 0) break; printf("[%d] vcache=0x%lx, cred=0x%lx, code=%d, refCount=%d, opcode=%d, flags=%x [%lx, %lx, %lx, %lx]\n", i, uentry->vnode, uentry->cred, uentry->code, uentry->refCount, uentry->opcode, uentry->flags, - uentry->parm[0], uentry->parm[1], uentry->parm[2], uentry->parm[3]); + uentry->size_parm[0], uentry->size_parm[1], uentry->ptr_parm[0], uentry->ptr_parm[1]); } printf("... found %d active 'afs_brs' entries\n", j);