diff --git a/sys/sparc64/fhc/fhc.c b/sys/sparc64/fhc/fhc.c index 8fcd25d120c6..f121b6ea97c4 100644 --- a/sys/sparc64/fhc/fhc.c +++ b/sys/sparc64/fhc/fhc.c @@ -74,7 +74,8 @@ static ofw_bus_get_devinfo_t fhc_get_devinfo; static void fhc_intr_enable(void *); static void fhc_intr_disable(void *); -static void fhc_intr_eoi(void *); +static void fhc_intr_assign(void *); +static void fhc_intr_clear(void *); static void fhc_led_func(void *, int); static int fhc_print_res(struct fhc_devinfo *); @@ -123,7 +124,8 @@ DRIVER_MODULE(fhc, nexus, fhc_driver, fhc_devclass, 0, 0); static const struct intr_controller fhc_ic = { fhc_intr_enable, fhc_intr_disable, - fhc_intr_eoi + fhc_intr_assign, + fhc_intr_clear }; struct fhc_icarg { @@ -366,7 +368,18 @@ fhc_intr_disable(void *arg) } static void -fhc_intr_eoi(void *arg) +fhc_intr_assign(void *arg) +{ + struct intr_vector *iv = arg; + struct fhc_icarg *fica = iv->iv_icarg; + + bus_write_4(fica->fica_memres, FHC_IMAP, INTMAP_TID( + bus_read_4(fica->fica_memres, FHC_IMAP), iv->iv_mid)); + (void)bus_read_4(fica->fica_memres, FHC_IMAP); +} + +static void +fhc_intr_clear(void *arg) { struct intr_vector *iv = arg; struct fhc_icarg *fica = iv->iv_icarg; diff --git a/sys/sparc64/include/bus_common.h b/sys/sparc64/include/bus_common.h index fce7584ea0ab..de6b22cbe6ea 100644 --- a/sys/sparc64/include/bus_common.h +++ b/sys/sparc64/include/bus_common.h @@ -56,10 +56,13 @@ #define INTSLOT(x) (((x) >> 3) & 0x7) #define INTPRI(x) ((x) & 0x7) #define INTINO(x) ((x) & INTMAP_INO_MASK) -#define INTMAP_ENABLE(mr, mid) \ - (((mr) & ~INTMAP_TID_MASK) | ((mid) << INTMAP_TID_SHIFT) | INTMAP_V) -#define INTMAP_VEC(ign, inr) \ - (((ign) << INTMAP_IGN_SHIFT) | (inr)) +#define INTMAP_ENABLE(mr, mid) \ + (INTMAP_TID((mr), (mid)) | INTMAP_V) +#define INTMAP_TID(mr, mid) \ + (((mr) & ~INTMAP_TID_MASK) | ((mid) << INTMAP_TID_SHIFT)) +#define INTMAP_VEC(ign, inr) \ + ((((ign) << INTMAP_IGN_SHIFT) & INTMAP_IGN_MASK) | \ + ((inr) & INTMAP_INR_MASK)) /* counter-timer support. */ void sparc64_counter_init(bus_space_tag_t tag, bus_space_handle_t handle, diff --git a/sys/sparc64/include/intr_machdep.h b/sys/sparc64/include/intr_machdep.h index 12badfbf943b..7f2980885b34 100644 --- a/sys/sparc64/include/intr_machdep.h +++ b/sys/sparc64/include/intr_machdep.h @@ -68,7 +68,8 @@ struct intr_request { struct intr_controller { void (*ic_enable)(void *); void (*ic_disable)(void *); - void (*ic_eoi)(void *); + void (*ic_assign)(void *); + void (*ic_clear)(void *); }; struct intr_vector { @@ -87,6 +88,10 @@ struct intr_vector { extern ih_func_t *intr_handlers[]; extern struct intr_vector intr_vectors[]; +#ifdef SMP +void intr_add_cpu(u_int cpu); +#endif +int intr_bind(int vec, u_char cpu); void intr_setup(int level, ih_func_t *ihf, int pri, iv_func_t *ivf, void *iva); void intr_init1(void); diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c index 8ffeb4894b66..87c9df8b491c 100644 --- a/sys/sparc64/pci/psycho.c +++ b/sys/sparc64/pci/psycho.c @@ -87,7 +87,8 @@ static int psycho_find_intrmap(struct psycho_softc *, u_int, bus_addr_t *, static driver_filter_t psycho_dmasync; static void psycho_intr_enable(void *); static void psycho_intr_disable(void *); -static void psycho_intr_eoi(void *); +static void psycho_intr_assign(void *); +static void psycho_intr_clear(void *); static bus_space_tag_t psycho_alloc_bus_tag(struct psycho_softc *, int); /* Interrupt handlers */ @@ -169,7 +170,8 @@ static SLIST_HEAD(, psycho_softc) psycho_softcs = static const struct intr_controller psycho_ic = { psycho_intr_enable, psycho_intr_disable, - psycho_intr_eoi + psycho_intr_assign, + psycho_intr_clear }; struct psycho_icarg { @@ -1099,7 +1101,17 @@ psycho_intr_disable(void *arg) } static void -psycho_intr_eoi(void *arg) +psycho_intr_assign(void *arg) +{ + struct intr_vector *iv = arg; + struct psycho_icarg *pica = iv->iv_icarg; + + PSYCHO_WRITE8(pica->pica_sc, pica->pica_map, INTMAP_TID( + PSYCHO_READ8(pica->pica_sc, pica->pica_map), iv->iv_mid)); +} + +static void +psycho_intr_clear(void *arg) { struct intr_vector *iv = arg; struct psycho_icarg *pica = iv->iv_icarg; diff --git a/sys/sparc64/sbus/sbus.c b/sys/sparc64/sbus/sbus.c index 44f51f55023a..3eac88087705 100644 --- a/sys/sparc64/sbus/sbus.c +++ b/sys/sparc64/sbus/sbus.c @@ -196,7 +196,8 @@ static struct sbus_devinfo * sbus_setup_dinfo(device_t, struct sbus_softc *, static void sbus_destroy_dinfo(struct sbus_devinfo *); static void sbus_intr_enable(void *); static void sbus_intr_disable(void *); -static void sbus_intr_eoi(void *); +static void sbus_intr_assign(void *); +static void sbus_intr_clear(void *); static int sbus_find_intrmap(struct sbus_softc *, u_int, bus_addr_t *, bus_addr_t *); static bus_space_tag_t sbus_alloc_bustag(struct sbus_softc *); @@ -253,7 +254,8 @@ DRIVER_MODULE(sbus, nexus, sbus_driver, sbus_devclass, 0, 0); static const struct intr_controller sbus_ic = { sbus_intr_enable, sbus_intr_disable, - sbus_intr_eoi + sbus_intr_assign, + sbus_intr_clear }; struct sbus_icarg { @@ -667,6 +669,7 @@ sbus_intr_enable(void *arg) SYSIO_WRITE8(sica->sica_sc, sica->sica_map, INTMAP_ENABLE(iv->iv_vec, iv->iv_mid)); } + static void sbus_intr_disable(void *arg) { @@ -677,7 +680,17 @@ sbus_intr_disable(void *arg) } static void -sbus_intr_eoi(void *arg) +sbus_intr_assign(void *arg) +{ + struct intr_vector *iv = arg; + struct sbus_icarg *sica = iv->iv_icarg; + + SYSIO_WRITE8(sica->sica_sc, sica->sica_map, INTMAP_TID( + SYSIO_READ8(sica->sica_sc, sica->sica_map), iv->iv_mid)); +} + +static void +sbus_intr_clear(void *arg) { struct intr_vector *iv = arg; struct sbus_icarg *sica = iv->iv_icarg; diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c index c771b6ad2f43..f3a1f444eb5b 100644 --- a/sys/sparc64/sparc64/intr_machdep.c +++ b/sys/sparc64/sparc64/intr_machdep.c @@ -63,16 +63,16 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include #include #include -#include +#include #include #include #include #include -#include +#include +#include #include #include @@ -102,24 +102,30 @@ static const char *pil_names[] = { }; /* protect the intr_vectors table */ -static struct mtx intr_table_lock; +static struct sx intr_table_lock; +/* protect intrcnt_index */ +static struct mtx intrcnt_lock; -static void intr_enable_eoi(void *); +#ifdef SMP +static int assign_cpu; + +static void intr_assign_next_cpu(struct intr_vector *iv); +#endif + +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_execute_handlers(void *); static void intr_stray_level(struct trapframe *); static void intr_stray_vector(void *); static int intrcnt_setname(const char *, int); static void intrcnt_updatename(int, const char *, int); -/* - * not MPSAFE - */ static void intrcnt_updatename(int vec, const char *name, int ispil) { static int intrcnt_index, stray_pil_index, stray_vec_index; int name_index; + mtx_lock_spin(&intrcnt_lock); if (intrnames[0] == '\0') { /* for bitbucket */ if (bootverbose) @@ -154,6 +160,7 @@ intrcnt_updatename(int vec, const char *name, int ispil) intr_countp[vec] = name_index; else pil_countp[vec] = name_index; + mtx_unlock_spin(&intrcnt_lock); } static int @@ -171,19 +178,19 @@ void intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva) { char pilname[MAXCOMLEN + 1]; - u_long ps; + register_t s; - ps = intr_disable(); + s = intr_disable(); if (vec != -1) { intr_vectors[vec].iv_func = ivf; intr_vectors[vec].iv_arg = iva; intr_vectors[vec].iv_pri = pri; intr_vectors[vec].iv_vec = vec; } + intr_handlers[pri] = ihf; + intr_restore(s); snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]); intrcnt_updatename(pri, pilname, 1); - intr_handlers[pri] = ihf; - intr_restore(ps); } static void @@ -230,19 +237,35 @@ void intr_init2() { - mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); + sx_init(&intr_table_lock, "intr sources"); + mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } -static void -intr_enable_eoi(void *arg) +static int +intr_assign_cpu(void *arg, u_char cpu) { +#ifdef SMP + struct pcpu *pc; struct intr_vector *iv; - const struct intr_controller *ic; - iv = arg; - ic = iv->iv_ic; - ic->ic_enable(iv); - ic->ic_eoi(iv); + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu && cpu != NOCPU) { + pc = pcpu_find(cpu); + if (pc == NULL) + return (EINVAL); + iv = arg; + sx_xlock(&intr_table_lock); + iv->iv_mid = pc->pc_mid; + iv->iv_ic->ic_assign(iv); + sx_xunlock(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif } static void @@ -263,32 +286,21 @@ intr_controller_register(int vec, const struct intr_controller *ic, struct intr_vector *iv; int error; + if (vec < 0 || vec >= IV_MAX) + return (EINVAL); + sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; - mtx_lock_spin(&intr_table_lock); ie = iv->iv_event; - mtx_unlock_spin(&intr_table_lock); + sx_xunlock(&intr_table_lock); if (ie != NULL) return (EEXIST); - /* - * Testing shows that at least with the interrupt controllers of - * Psycho and Schizo bridges enabling an interrupt doesn't cause - * an outstanding interrupt to be issued to the CPU. Thus we can't - * use a function doing disable+EOI for the "disable" pointer as - * done on other architectures because this would lead to a lost - * interrupt if it triggers while we are still processing the - * previous one. Instead we use an enable+EOI approach because as - * outlined in the Tomatillo documentation clearing an interrupt - * in the interrupt controller causes it to be (re)issued to the - * CPU as long as the source of a level sensitive interrupt is - * not cleared. - */ - error = intr_event_create(&ie, iv, 0, vec, NULL, intr_enable_eoi, - intr_enable_eoi, NULL, "vec%d:", vec); + error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear, + ic->ic_clear, intr_assign_cpu, "vec%d:", vec); if (error != 0) return (error); - mtx_lock_spin(&intr_table_lock); + sx_xlock(&intr_table_lock); if (iv->iv_event != NULL) { - mtx_unlock_spin(&intr_table_lock); + sx_xunlock(&intr_table_lock); intr_event_destroy(ie); return (EEXIST); } @@ -296,7 +308,7 @@ intr_controller_register(int vec, const struct intr_controller *ic, iv->iv_icarg = icarg; iv->iv_event = ie; iv->iv_mid = PCPU_GET(mid); - mtx_unlock_spin(&intr_table_lock); + sx_xunlock(&intr_table_lock); return (0); } @@ -310,20 +322,20 @@ inthand_add(const char *name, int vec, driver_filter_t *filt, struct intr_vector *iv; int error, fast; + if (vec < 0 || vec >= IV_MAX) + return (EINVAL); + sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; - mtx_lock_spin(&intr_table_lock); ic = iv->iv_ic; ie = iv->iv_event; - mtx_unlock_spin(&intr_table_lock); + sx_xunlock(&intr_table_lock); if (ic == NULL || ie == NULL) return (EINVAL); - error = intr_event_add_handler(ie, name, filt, handler, arg, intr_priority(flags), flags, cookiep); if (error != 0) return (error); - - mtx_lock_spin(&intr_table_lock); + sx_xlock(&intr_table_lock); /* Disable the interrupt while we fiddle with it. */ ic->ic_disable(iv); iv->iv_refcnt++; @@ -350,9 +362,14 @@ inthand_add(const char *name, int vec, driver_filter_t *filt, } intr_stray_count[vec] = 0; intrcnt_updatename(vec, ie->ie_fullname, 0); +#ifdef SMP + if (assign_cpu) + intr_assign_next_cpu(iv); +#endif + ic->ic_enable(iv); /* Ensure the interrupt is cleared, it might have triggered before. */ - intr_enable_eoi(iv); - mtx_unlock_spin(&intr_table_lock); + ic->ic_clear(iv); + sx_xunlock(&intr_table_lock); return (0); } @@ -362,14 +379,16 @@ inthand_remove(int vec, void *cookie) struct intr_vector *iv; int error; + if (vec < 0 || vec >= IV_MAX) + return (EINVAL); error = intr_event_remove_handler(cookie); if (error == 0) { /* * XXX: maybe this should be done regardless of whether * intr_event_remove_handler() succeeded? */ + sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; - mtx_lock_spin(&intr_table_lock); iv->iv_refcnt--; if (iv->iv_refcnt == 0) { /* @@ -379,7 +398,109 @@ inthand_remove(int vec, void *cookie) intr_setup(PIL_LOW, intr_fast, vec, intr_stray_vector, iv); } - mtx_unlock_spin(&intr_table_lock); + sx_xunlock(&intr_table_lock); } return (error); } + +#ifdef SMP +/* + * Support for balancing interrupt sources across CPUs. For now we just + * allocate CPUs round-robin. + */ + +/* The BSP is always a valid target. */ +static cpumask_t intr_cpus = (1 << 0); +static int current_cpu; + +static void +intr_assign_next_cpu(struct intr_vector *iv) +{ + struct pcpu *pc; + + sx_assert(&intr_table_lock, SA_XLOCKED); + + /* + * Assign this source to a CPU in a round-robin fashion. + */ + pc = pcpu_find(current_cpu); + if (pc == NULL) + return; + iv->iv_mid = pc->pc_mid; + iv->iv_ic->ic_assign(iv); + do { + current_cpu++; + if (current_cpu > mp_maxid) + current_cpu = 0; + } while (!(intr_cpus & (1 << current_cpu))); +} + +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(int vec, u_char cpu) +{ + struct intr_vector *iv; + + if (vec < 0 || vec >= IV_MAX) + return (EINVAL); + iv = &intr_vectors[vec]; + if (iv == NULL) + return (EINVAL); + return (intr_event_bind(iv->iv_event, cpu)); +} + +/* + * Add a CPU to our mask of valid CPUs that can be destinations of + * interrupts. + */ +void +intr_add_cpu(u_int cpu) +{ + + if (cpu >= MAXCPU) + panic("%s: Invalid CPU ID", __func__); + if (bootverbose) + printf("INTR: Adding CPU %d as a target\n", cpu); + + intr_cpus |= (1 << cpu); +} + +/* + * Distribute all the interrupt sources among the available CPUs once the + * AP's have been launched. + */ +static void +intr_shuffle_irqs(void *arg __unused) +{ + struct pcpu *pc; + struct intr_vector *iv; + int i; + + /* Don't bother on UP. */ + if (mp_ncpus == 1) + return; + + /* Round-robin assign a CPU to each enabled source. */ + sx_xlock(&intr_table_lock); + assign_cpu = 1; + for (i = 0; i < IV_MAX; i++) { + iv = &intr_vectors[i]; + if (iv != NULL && iv->iv_refcnt > 0) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (iv->iv_event->ie_cpu != NOCPU && + (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) { + iv->iv_mid = pc->pc_mid; + iv->iv_ic->ic_assign(iv); + } else + intr_assign_next_cpu(iv); + } + } + sx_xunlock(&intr_table_lock); +} +SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, + NULL); +#endif diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c index d9bebcca761a..2056b306d500 100644 --- a/sys/sparc64/sparc64/mp_machdep.c +++ b/sys/sparc64/sparc64/mp_machdep.c @@ -171,7 +171,7 @@ cpu_mp_setmaxid(void) { char buf[128]; phandle_t child; - int cpus; + u_int cpus; all_cpus = 1 << curcpu; mp_ncpus = 1; @@ -251,7 +251,7 @@ cpu_mp_start(void) phandle_t child; u_int clock; u_int mid; - int cpuid; + u_int cpuid; mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); @@ -302,6 +302,7 @@ cpu_mp_start(void) pc->pc_node = child; all_cpus |= 1 << cpuid; + intr_add_cpu(cpuid); } KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, ("%s: can only IPI a maximum of %d JBus-CPUs", diff --git a/sys/sparc64/sparc64/nexus.c b/sys/sparc64/sparc64/nexus.c index 23314d86038d..7a16ba5d1dad 100644 --- a/sys/sparc64/sparc64/nexus.c +++ b/sys/sparc64/sparc64/nexus.c @@ -93,6 +93,9 @@ static bus_get_resource_list_t nexus_get_resource_list; static bus_get_dma_tag_t nexus_get_dma_tag; static ofw_bus_get_devinfo_t nexus_get_devinfo; +#ifdef SMP +static int nexus_bind_intr(device_t, device_t, struct resource *, int); +#endif static int nexus_inlist(const char *, const char **); static struct nexus_devinfo * nexus_setup_dinfo(device_t, phandle_t); static void nexus_destroy_dinfo(struct nexus_devinfo *); @@ -119,6 +122,9 @@ static device_method_t nexus_methods[] = { DEVMETHOD(bus_release_resource, nexus_release_resource), DEVMETHOD(bus_setup_intr, nexus_setup_intr), DEVMETHOD(bus_teardown_intr, nexus_teardown_intr), +#ifdef SMP + DEVMETHOD(bus_bind_intr, nexus_bind_intr), +#endif DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_get_resource_list, nexus_get_resource_list), DEVMETHOD(bus_get_dma_tag, nexus_get_dma_tag), @@ -311,6 +317,15 @@ nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih) return (0); } +#ifdef SMP +static int +nexus_bind_intr(device_t dev, device_t child, struct resource *r, int cpu) +{ + + return (intr_bind(rman_get_start(r), cpu)); +} +#endif + static struct resource * nexus_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) diff --git a/sys/sparc64/sparc64/upa.c b/sys/sparc64/sparc64/upa.c index 3a21527d17ae..5d931e1560d7 100644 --- a/sys/sparc64/sparc64/upa.c +++ b/sys/sparc64/sparc64/upa.c @@ -108,6 +108,7 @@ static ofw_bus_get_devinfo_t upa_get_devinfo; static void upa_intr_enable(void *); static void upa_intr_disable(void *); +static void upa_intr_assign(void *); static struct upa_devinfo *upa_setup_dinfo(device_t, struct upa_softc *, phandle_t, uint32_t); static void upa_destroy_dinfo(struct upa_devinfo *); @@ -154,6 +155,7 @@ DRIVER_MODULE(upa, nexus, upa_driver, upa_devclass, 0, 0); static const struct intr_controller upa_ic = { upa_intr_enable, upa_intr_disable, + upa_intr_assign, /* The interrupts are pulse type and thus automatically cleared. */ NULL }; @@ -469,6 +471,17 @@ upa_intr_disable(void *arg) (void)UPA_READ(uica->uica_sc, uica->uica_imr, 0x0); } +static void +upa_intr_assign(void *arg) +{ + struct intr_vector *iv = arg; + struct upa_icarg *uica = iv->iv_icarg; + + UPA_WRITE(uica->uica_sc, uica->uica_imr, 0x0, INTMAP_TID( + UPA_READ(uica->uica_sc, uica->uica_imr, 0x0), iv->iv_mid)); + (void)UPA_READ(uica->uica_sc, uica->uica_imr, 0x0); +} + static int upa_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *func, void *arg, void **cookiep)