On 9 Jul, I wrote: > The patch below enhances the mutex pool code to support the creation and > use of multiple mutex pools. It creates one pool of sleep mutexes with > the MTX_NOWITNESS flag for use in building higher level (sx and lockmgr) > locks. It also creates another pool without MTX_NOWITNESS for general > purpose use. It can also be used to create pools of spin mutexes. > > The users of the existing MTX_NOWITNESS pool are modified to use the > appropriate pool. > > If this had been implemented earlier, it would have allowed witness to > catch the Giant vs. FILE_LOCK() lock order reversal that I recently > tracked down and fixed. > > I've tested this only on i386, but it also passes the "make universe" > test. It might help to actually include the patch ... Index: sys/kern/kern_descrip.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_descrip.c,v retrieving revision 1.207 diff -u -r1.207 kern_descrip.c --- sys/kern/kern_descrip.c 4 Jul 2003 12:20:27 -0000 1.207 +++ sys/kern/kern_descrip.c 5 Jul 2003 22:21:42 -0000 _at__at_ -1209,7 +1209,7 _at__at_ * descriptor to the list of open files at that point, otherwise * put it at the front of the list of open files. */ - fp->f_mtxp = mtx_pool_alloc(); + fp->f_mtxp = mtx_pool_alloc(mtxpool_sleep); fp->f_count = 1; fp->f_cred = crhold(td->td_ucred); fp->f_ops = &badfileops; Index: sys/kern/kern_lock.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_lock.c,v retrieving revision 1.68 diff -u -r1.68 kern_lock.c --- sys/kern/kern_lock.c 11 Jun 2003 00:56:56 -0000 1.68 +++ sys/kern/kern_lock.c 5 Jul 2003 20:28:33 -0000 _at__at_ -547,9 +547,9 _at__at_ * XXX cleanup - make sure mtxpool is always initialized before * this is ever called. */ - if (mtx_pool_valid) { + if (mtxpool_lockbuilder != NULL) { mtx_lock(&lock_mtx); - lkp->lk_interlock = mtx_pool_alloc(); + lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder); mtx_unlock(&lock_mtx); } else { lkp->lk_interlock = &lock_mtx; Index: sys/kern/kern_mtxpool.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_mtxpool.c,v retrieving revision 1.7 diff -u -r1.7 kern_mtxpool.c --- sys/kern/kern_mtxpool.c 13 Jun 2003 19:39:21 -0000 1.7 +++ sys/kern/kern_mtxpool.c 9 Jul 2003 05:10:51 -0000 _at__at_ -35,85 +35,164 _at__at_ #include <sys/mutex.h> #include <sys/systm.h> -#ifndef MTX_POOL_SIZE -#define MTX_POOL_SIZE 128 -#endif -#define MTX_POOL_MASK (MTX_POOL_SIZE - 1) -static struct mtx mtx_pool_ary[MTX_POOL_SIZE]; +MALLOC_DEFINE(M_MTXPOOL, "mtx_pool", "mutex pool"); -int mtx_pool_valid = 0; +/* Pool sizes must be a power of two */ +#ifndef MTX_POOL_LOCKBUILDER_SIZE +#define MTX_POOL_LOCKBUILDER_SIZE 128 +#endif +#ifndef MTX_POOL_SLEEP_SIZE +#define MTX_POOL_SLEEP_SIZE 128 +#endif + +struct mtxpool_header { + int mtxpool_size; + int mtxpool_mask; + int mtxpool_shift; + int mtxpool_next; +}; + +struct mtx_pool { + struct mtxpool_header mtx_pool_header; + struct mtx mtx_pool_ary[1]; +}; + +static struct mtx_pool_lockbuilder { + struct mtxpool_header mtx_pool_header; + struct mtx mtx_pool_ary[MTX_POOL_LOCKBUILDER_SIZE]; +} lockbuilder_pool; + +#define mtx_pool_size mtx_pool_header.mtxpool_size +#define mtx_pool_mask mtx_pool_header.mtxpool_mask +#define mtx_pool_shift mtx_pool_header.mtxpool_shift +#define mtx_pool_next mtx_pool_header.mtxpool_next + +struct mtx_pool *mtxpool_sleep; +struct mtx_pool *mtxpool_lockbuilder; + +#if UINTPTR_MAX == UINT64_MAX /* 64 bits */ +# define POINTER_BITS 64 +# define HASH_MULTIPLIER 11400714819323198485u /* (2^64)*(sqrt(5)-1)/2 */ +#else /* assume 32 bits */ +# define POINTER_BITS 32 +# define HASH_MULTIPLIER 2654435769u /* (2^32)*(sqrt(5)-1)/2 */ +#endif /* - * Inline version of mtx_pool_find(), used to streamline our main API - * function calls. + * Return the (shared) pool mutex associated with the specified address. + * The returned mutex is a leaf level mutex, meaning that if you obtain it + * you cannot obtain any other mutexes until you release it. You can + * legally msleep() on the mutex. */ -static __inline struct mtx * -_mtx_pool_find(void *ptr) +struct mtx * +mtx_pool_find(struct mtx_pool *pool, void *ptr) { - int p; + int p; - p = (int)(uintptr_t)ptr; - return (&mtx_pool_ary[(p ^ (p >> 6)) & MTX_POOL_MASK]); + KASSERT(pool != NULL, ("_mtx_pool_find(): null pool")); + /* + * Fibonacci hash, see Knuth's + * _Art of Computer Programming, Volume 3 / Sorting and Searching_ + */ + p = ((HASH_MULTIPLIER * (uintptr_t)ptr) >> pool->mtx_pool_shift) & + pool->mtx_pool_mask; + return (&pool->mtx_pool_ary[p]); } static void -mtx_pool_setup(void *dummy __unused) +mtx_pool_initialize(struct mtx_pool *pool, const char *mtx_name, int pool_size, + int opts) { - int i; + int i, maskbits; - for (i = 0; i < MTX_POOL_SIZE; ++i) - mtx_init(&mtx_pool_ary[i], "pool mutex", NULL, - MTX_DEF | MTX_NOWITNESS | MTX_QUIET); - mtx_pool_valid = 1; + pool->mtx_pool_size = pool_size; + pool->mtx_pool_mask = pool_size - 1; + for (i = 1, maskbits = 0; (i & pool_size) == 0; i = i << 1) + maskbits++; + pool->mtx_pool_shift = POINTER_BITS - maskbits; + pool->mtx_pool_next = 0; + for (i = 0; i < pool_size; ++i) + mtx_init(&pool->mtx_pool_ary[i], mtx_name, NULL, opts); +} + +struct mtx_pool * +mtx_pool_create(const char *mtx_name, int pool_size, int opts) +{ + struct mtx_pool *pool; + + if (pool_size <= 0 || !powerof2(pool_size)) { + printf("WARNING: %s pool size is not a power of 2.\n", + mtx_name); + pool_size = 128; + } + MALLOC(pool, struct mtx_pool *, + sizeof (struct mtx_pool) + ((pool_size - 1) * sizeof (struct mtx)), + M_MTXPOOL, M_WAITOK | M_ZERO); + mtx_pool_initialize(pool, mtx_name, pool_size, opts); + return pool; } -/* - * Obtain a (shared) mutex from the pool. The returned mutex is a leaf - * level mutex, meaning that if you obtain it you cannot obtain any other - * mutexes until you release it. You can legally msleep() on the mutex. - */ -struct mtx * -mtx_pool_alloc(void) +void +mtx_pool_destroy(struct mtx_pool **poolp) { - static int si; + int i; + struct mtx_pool *pool = *poolp; - return (&mtx_pool_ary[si++ & MTX_POOL_MASK]); + for (i = pool->mtx_pool_size - 1; i >= 0; --i) + mtx_destroy(&pool->mtx_pool_ary[i]); + FREE(pool, M_MTXPOOL); + *poolp = NULL; } -/* - * Return the (shared) pool mutex associated with the specified address. - * The returned mutex is a leaf level mutex, meaning that if you obtain it - * you cannot obtain any other mutexes until you release it. You can - * legally msleep() on the mutex. - */ -struct mtx * -mtx_pool_find(void *ptr) +static void +mtx_pool_setup_static(void *dummy __unused) { - - return (_mtx_pool_find(ptr)); + mtx_pool_initialize((struct mtx_pool *)&lockbuilder_pool, + "lockbuilder mtxpool", MTX_POOL_LOCKBUILDER_SIZE, + MTX_DEF | MTX_NOWITNESS | MTX_QUIET); + mtxpool_lockbuilder = (struct mtx_pool *)&lockbuilder_pool; } -/* - * Combined find/lock operation. Lock the pool mutex associated with - * the specified address. - */ -void -mtx_pool_lock(void *ptr) +static void +mtx_pool_setup_dynamic(void *dummy __unused) { - - mtx_lock(_mtx_pool_find(ptr)); + mtxpool_sleep = mtx_pool_create("sleep mtxpool", + MTX_POOL_SLEEP_SIZE, MTX_DEF); } /* - * Combined find/unlock operation. Unlock the pool mutex associated with - * the specified address. + * Obtain a (shared) mutex from the pool. The returned mutex is a leaf + * level mutex, meaning that if you obtain it you cannot obtain any other + * mutexes until you release it. You can legally msleep() on the mutex. */ -void -mtx_pool_unlock(void *ptr) +struct mtx * +mtx_pool_alloc(struct mtx_pool *pool) { + int i; - mtx_unlock(_mtx_pool_find(ptr)); + KASSERT(pool != NULL, ("mtx_pool_alloc(): null pool")); + /* + * mtx_pool_next is unprotected against multiple accesses, + * but simultaneous access by two CPUs should not be very + * harmful. + */ + i = pool->mtx_pool_next; + pool->mtx_pool_next = (i + 1) & pool->mtx_pool_mask; + return (&pool->mtx_pool_ary[i]); } -SYSINIT(mtxpooli, SI_SUB_MTX_POOL, SI_ORDER_FIRST, mtx_pool_setup, NULL); +/* + * The lockbuilder pool must be initialized early because the lockmgr + * and sx locks depend on it. The sx locks are used in the kernel + * memory allocator. The lockmgr subsystem is initialized by + * SYSINIT(..., SI_SUB_LOCK, ...). + * + * We can't call MALLOC() to dynamically allocate the sleep pool + * until after kmeminit() has been called, which is done by + * SYSINIT(..., SI_SUB_KMEM, ...). + */ +SYSINIT(mtxpooli1, SI_SUB_MTX_POOL_STATIC, SI_ORDER_FIRST, + mtx_pool_setup_static, NULL); +SYSINIT(mtxpooli2, SI_SUB_MTX_POOL_DYNAMIC, SI_ORDER_FIRST, + mtx_pool_setup_dynamic, NULL); Index: sys/kern/kern_prot.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_prot.c,v retrieving revision 1.174 diff -u -r1.174 kern_prot.c --- sys/kern/kern_prot.c 4 Jul 2003 02:21:28 -0000 1.174 +++ sys/kern/kern_prot.c 5 Jul 2003 22:21:42 -0000 _at__at_ -1652,7 +1652,7 _at__at_ MALLOC(cr, struct ucred *, sizeof(*cr), M_CRED, M_WAITOK | M_ZERO); cr->cr_ref = 1; - cr->cr_mtxp = mtx_pool_find(cr); + cr->cr_mtxp = mtx_pool_find(mtxpool_sleep, cr); #ifdef MAC mac_init_cred(cr); #endif Index: sys/kern/kern_resource.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_resource.c,v retrieving revision 1.126 diff -u -r1.126 kern_resource.c --- sys/kern/kern_resource.c 11 Jun 2003 00:56:56 -0000 1.126 +++ sys/kern/kern_resource.c 5 Jul 2003 03:21:48 -0000 _at__at_ -893,7 +893,7 _at__at_ free(uip, M_UIDINFO); uip = old_uip; } else { - uip->ui_mtxp = mtx_pool_alloc(); + uip->ui_mtxp = mtx_pool_alloc(mtxpool_sleep); uip->ui_uid = uid; LIST_INSERT_HEAD(UIHASH(uid), uip, ui_hash); } Index: sys/kern/kern_sx.c =================================================================== RCS file: /home/ncvs/src/sys/kern/kern_sx.c,v retrieving revision 1.19 diff -u -r1.19 kern_sx.c --- sys/kern/kern_sx.c 11 Jun 2003 00:56:56 -0000 1.19 +++ sys/kern/kern_sx.c 5 Jul 2003 03:22:11 -0000 _at__at_ -74,7 +74,7 _at__at_ lock->lo_type = lock->lo_name = description; lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE; - sx->sx_lock = mtx_pool_find(sx); + sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); sx->sx_cnt = 0; cv_init(&sx->sx_shrd_cv, description); sx->sx_shrd_wcnt = 0; Index: sys/sys/kernel.h =================================================================== RCS file: /home/ncvs/src/sys/sys/kernel.h,v retrieving revision 1.110 diff -u -r1.110 kernel.h --- sys/sys/kernel.h 3 Jun 2003 08:41:04 -0000 1.110 +++ sys/sys/kernel.h 5 Jul 2003 20:47:40 -0000 _at__at_ -112,11 +112,12 _at__at_ SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */ SI_SUB_CONSOLE = 0x0800000, /* console*/ SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/ - SI_SUB_MTX_POOL = 0x0900000, /* mutex pool */ + SI_SUB_MTX_POOL_STATIC = 0x0900000, /* static mutex pool */ SI_SUB_VM = 0x1000000, /* virtual memory system init*/ SI_SUB_KMEM = 0x1800000, /* kernel memory*/ SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/ SI_SUB_WITNESS = 0x1A80000, /* witness initialization */ + SI_SUB_MTX_POOL_DYNAMIC = 0x1AC0000, /* dynamic mutex pool */ SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */ SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */ SI_SUB_KLD = 0x2000000, /* KLD and module setup */ Index: sys/sys/mutex.h =================================================================== RCS file: /home/ncvs/src/sys/sys/mutex.h,v retrieving revision 1.61 diff -u -r1.61 mutex.h --- sys/sys/mutex.h 18 May 2003 03:46:30 -0000 1.61 +++ sys/sys/mutex.h 9 Jul 2003 02:04:05 -0000 _at__at_ -236,12 +236,30 _at__at_ #define mtx_unlock(m) mtx_unlock_flags((m), 0) #define mtx_unlock_spin(m) mtx_unlock_spin_flags((m), 0) -struct mtx *mtx_pool_find(void *ptr); -struct mtx *mtx_pool_alloc(void); -void mtx_pool_lock(void *ptr); -void mtx_pool_unlock(void *ptr); +struct mtx_pool; -extern int mtx_pool_valid; +struct mtx_pool *mtx_pool_create(const char *mtx_name, int pool_size, int opts); +void mtx_pool_destroy(struct mtx_pool **poolp); +struct mtx *mtx_pool_find(struct mtx_pool *pool, void *ptr); +struct mtx *mtx_pool_alloc(struct mtx_pool *pool); +struct mtx *mtx_pool_alloc_spin(struct mtx_pool *pool); +#define mtx_pool_lock(pool, ptr) \ + mtx_lock(mtx_pool_find((pool), (ptr))) +#define mtx_pool_lock_spin(pool, ptr) \ + mtx_lock_spin(mtx_pool_find((pool), (ptr))) +#define mtx_pool_unlock(pool, ptr) \ + mtx_unlock(mtx_pool_find((pool), (ptr))) +#define mtx_pool_unlock_spin(pool, ptr) \ + mtx_unlock_spin(mtx_pool_find((pool), (ptr))) + +/* + * mtxpool_lockbuilder is a pool of sleep locks that is not witness + * checked and should only be used for building higher level locks. + * + * mtxpool_sleep is a general purpose pool of sleep mutexes. + */ +extern struct mtx_pool *mtxpool_lockbuilder; +extern struct mtx_pool *mtxpool_sleep; #ifndef LOCK_DEBUG #error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/mutex.h>Received on Wed Jul 09 2003 - 09:43:02 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:37:14 UTC