diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c index 23778fbd8962..3262029150e8 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c @@ -421,7 +421,7 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes) vm_page_reference(pp); vm_page_lock(pp); zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb"); + vm_page_busy_sleep(pp, "zfsmwb", true); zfs_vmobject_wlock(obj); continue; } @@ -476,7 +476,7 @@ page_hold(vnode_t *vp, int64_t start) vm_page_reference(pp); vm_page_lock(pp); zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb"); + vm_page_busy_sleep(pp, "zfsmwb", true); zfs_vmobject_wlock(obj); continue; } diff --git a/sys/dev/drm2/i915/i915_gem.c b/sys/dev/drm2/i915/i915_gem.c index 511400844580..4624997eea08 100644 --- a/sys/dev/drm2/i915/i915_gem.c +++ b/sys/dev/drm2/i915/i915_gem.c @@ -1976,7 +1976,7 @@ retry: DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_busy_sleep(m, "915pee"); + vm_page_busy_sleep(m, "915pee", false); goto retry; } goto have_page; @@ -2037,7 +2037,7 @@ retry: DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_busy_sleep(m, "915pbs"); + vm_page_busy_sleep(m, "915pbs", false); goto retry; } if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c index 9c9844820ad1..f172a030996a 100644 --- a/sys/dev/drm2/ttm/ttm_bo_vm.c +++ b/sys/dev/drm2/ttm/ttm_bo_vm.c @@ -239,7 +239,7 @@ reserve: if (vm_page_busied(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_busy_sleep(m, "ttmpbs"); + vm_page_busy_sleep(m, "ttmpbs", false); VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index f728859801f0..f28db3e638e4 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -1724,7 +1724,7 @@ brelse(struct buf *bp) while (vm_page_xbusied(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(obj); - vm_page_busy_sleep(m, "mbncsh"); + vm_page_busy_sleep(m, "mbncsh", true); VM_OBJECT_WLOCK(obj); } if (pmap_page_wired_mappings(m) == 0) @@ -4069,7 +4069,7 @@ vfs_drain_busy_pages(struct buf *bp) while (vm_page_xbusied(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); - vm_page_busy_sleep(m, "vbpage"); + vm_page_busy_sleep(m, "vbpage", true); VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); } } diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index b2a7013b883a..d4c71169555c 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -1182,7 +1182,7 @@ shadowlookup: if (object != tobject) VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(tobject); - vm_page_busy_sleep(m, "madvpo"); + vm_page_busy_sleep(m, "madvpo", false); VM_OBJECT_WLOCK(object); goto relookup; } @@ -1361,7 +1361,7 @@ retry: VM_OBJECT_WUNLOCK(new_object); vm_page_lock(m); VM_OBJECT_WUNLOCK(orig_object); - vm_page_busy_sleep(m, "spltwt"); + vm_page_busy_sleep(m, "spltwt", false); VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; @@ -1450,7 +1450,7 @@ vm_object_backing_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next, if (p == NULL) VM_WAIT; else - vm_page_busy_sleep(p, "vmocol"); + vm_page_busy_sleep(p, "vmocol", false); VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(backing_object); return (TAILQ_FIRST(&backing_object->memq)); @@ -1917,7 +1917,7 @@ again: vm_page_lock(p); if (vm_page_xbusied(p)) { VM_OBJECT_WUNLOCK(object); - vm_page_busy_sleep(p, "vmopax"); + vm_page_busy_sleep(p, "vmopax", true); VM_OBJECT_WLOCK(object); goto again; } @@ -1932,7 +1932,7 @@ again: } if (vm_page_busied(p)) { VM_OBJECT_WUNLOCK(object); - vm_page_busy_sleep(p, "vmopar"); + vm_page_busy_sleep(p, "vmopar", false); VM_OBJECT_WLOCK(object); goto again; } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index f166649d5755..054148bc93e7 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -601,21 +601,20 @@ vm_page_sunbusy(vm_page_t m) * This is used to implement the hard-path of busying mechanism. * * The given page must be locked. + * + * If nonshared is true, sleep only if the page is xbusy. */ void -vm_page_busy_sleep(vm_page_t m, const char *wmesg) +vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared) { u_int x; - vm_page_lock_assert(m, MA_OWNED); + vm_page_assert_locked(m); x = m->busy_lock; - if (x == VPB_UNBUSIED) { - vm_page_unlock(m); - return; - } - if ((x & VPB_BIT_WAITERS) == 0 && - !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS)) { + if (x == VPB_UNBUSIED || (nonshared && (x & VPB_BIT_SHARED) != 0) || + ((x & VPB_BIT_WAITERS) == 0 && + !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS))) { vm_page_unlock(m); return; } @@ -928,7 +927,7 @@ vm_page_sleep_if_busy(vm_page_t m, const char *msg) obj = m->object; vm_page_lock(m); VM_OBJECT_WUNLOCK(obj); - vm_page_busy_sleep(m, msg); + vm_page_busy_sleep(m, msg, false); VM_OBJECT_WLOCK(obj); return (TRUE); } @@ -2717,7 +2716,8 @@ retrylookup: vm_page_aflag_set(m, PGA_REFERENCED); vm_page_lock(m); VM_OBJECT_WUNLOCK(object); - vm_page_busy_sleep(m, "pgrbwt"); + vm_page_busy_sleep(m, "pgrbwt", (allocflags & + VM_ALLOC_IGN_SBUSY) != 0); VM_OBJECT_WLOCK(object); goto retrylookup; } else { diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 3ab4c24e1c1d..7cf949e9d2f2 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -420,7 +420,7 @@ malloc2vm_flags(int malloc_flags) #endif void vm_page_busy_downgrade(vm_page_t m); -void vm_page_busy_sleep(vm_page_t m, const char *msg); +void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared); void vm_page_flash(vm_page_t m); void vm_page_hold(vm_page_t mem); void vm_page_unhold(vm_page_t mem);