callout: Move per-CPU callout state into the dpcpu region

This eliminates some static bloat in amd64 kernels and reduces the
penalty of increasing MAXCPU.  The structures now also maintain NUMA
affinity.  No functional change intended.

PR:		269572
Reviewed by:	mjg, kib
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D39807
This commit is contained in:
Mark Johnston 2023-04-26 10:09:09 -04:00
parent 74ac712f72
commit 78cfa762eb

View File

@ -195,9 +195,9 @@ struct callout_cpu {
#define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time
#define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec
static struct callout_cpu cc_cpu[MAXCPU];
DPCPU_DEFINE_STATIC(struct callout_cpu, cc_cpu);
#define CPUBLOCK MAXCPU
#define CC_CPU(cpu) (&cc_cpu[(cpu)])
#define CC_CPU(cpu) DPCPU_ID_PTR(cpu, cc_cpu)
#define CC_SELF() CC_CPU(PCPU_GET(cpuid))
#else
static struct callout_cpu cc_cpu;
@ -321,7 +321,7 @@ callout_cpu_init(struct callout_cpu *cc, int cpu)
{
int i;
mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN);
mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_NEW);
cc->cc_callwheel = malloc_domainset(sizeof(struct callout_list) *
callwheelsize, M_CALLOUT,
DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), M_WAITOK);