o Rename ic_eoi to ic_clear to emphasize the functions it points

don't send and EOI which works like on amd64/i386 and blocks all
  interrupts on the relevant interrupt controller.
o Replace the post_filter and post_inthread hooks registered when
  creating the interrupt events with just ic_clear as on sparc64 we
  don't need to do any disable->EOI->enable dance to unblock all but
  the relevant interrupt while running the filter or handler; just
  not clearing the interrupt already has the same effect.
o Merge from amd64/i386:
  - Split the intr_table_lock into an sx lock used for most things,
    and a spin lock to protect intrcnt_index.
  - Add support for binding interrupts to CPUs, including for the
    bus_bind_intr(9) interface, a assign_cpu hook and initially
    shuffling interrupts arround in a round-robin fashion.

Reviewed by:	jhb
MFC after:	1 month

sys/sparc64/fhc/fhc.c

@@ -74,7 +74,8 @@ static ofw_bus_get_devinfo_t fhc_get_devinfo;
 
 static void fhc_intr_enable(void *);
 static void fhc_intr_disable(void *);
-static void fhc_intr_eoi(void *);
+static void fhc_intr_assign(void *);
+static void fhc_intr_clear(void *);
 static void fhc_led_func(void *, int);
 static int fhc_print_res(struct fhc_devinfo *);
 
@@ -123,7 +124,8 @@ DRIVER_MODULE(fhc, nexus, fhc_driver, fhc_devclass, 0, 0);
 static const struct intr_controller fhc_ic = {
 	fhc_intr_enable,
 	fhc_intr_disable,
-	fhc_intr_eoi
+	fhc_intr_assign,
+	fhc_intr_clear
 };
 
 struct fhc_icarg {
@@ -366,7 +368,18 @@ fhc_intr_disable(void *arg)
 }
 
 static void
-fhc_intr_eoi(void *arg)
+fhc_intr_assign(void *arg)
+{
+	struct intr_vector *iv = arg;
+	struct fhc_icarg *fica = iv->iv_icarg;
+
+	bus_write_4(fica->fica_memres, FHC_IMAP, INTMAP_TID(
+	    bus_read_4(fica->fica_memres, FHC_IMAP), iv->iv_mid));
+	(void)bus_read_4(fica->fica_memres, FHC_IMAP);
+}
+
+static void
+fhc_intr_clear(void *arg)
 {
 	struct intr_vector *iv = arg;
 	struct fhc_icarg *fica = iv->iv_icarg;

sys/sparc64/include/bus_common.h

@@ -56,10 +56,13 @@
 #define	INTSLOT(x)		(((x) >> 3) & 0x7)
 #define	INTPRI(x)		((x) & 0x7)
 #define	INTINO(x)		((x) & INTMAP_INO_MASK)
-#define	INTMAP_ENABLE(mr, mid) \
-	(((mr) & ~INTMAP_TID_MASK) | ((mid) << INTMAP_TID_SHIFT) | INTMAP_V)
-#define	INTMAP_VEC(ign, inr) \
-	(((ign) << INTMAP_IGN_SHIFT) | (inr))
+#define	INTMAP_ENABLE(mr, mid)						\
+	(INTMAP_TID((mr), (mid)) | INTMAP_V)
+#define	INTMAP_TID(mr, mid)						\
+	(((mr) & ~INTMAP_TID_MASK) | ((mid) << INTMAP_TID_SHIFT))
+#define	INTMAP_VEC(ign, inr)						\
+	((((ign) << INTMAP_IGN_SHIFT) & INTMAP_IGN_MASK) |		\
+	((inr) & INTMAP_INR_MASK))
 
 /* counter-timer support. */
 void sparc64_counter_init(bus_space_tag_t tag, bus_space_handle_t handle,

sys/sparc64/include/intr_machdep.h

@@ -68,7 +68,8 @@ struct intr_request {
 struct intr_controller {
 	void	(*ic_enable)(void *);
 	void	(*ic_disable)(void *);
-	void	(*ic_eoi)(void *);
+	void	(*ic_assign)(void *);
+	void	(*ic_clear)(void *);
 };
 
 struct intr_vector {
@@ -87,6 +88,10 @@ struct intr_vector {
 extern ih_func_t *intr_handlers[];
 extern struct intr_vector intr_vectors[];
 
+#ifdef SMP
+void	intr_add_cpu(u_int cpu);
+#endif
+int	intr_bind(int vec, u_char cpu);
 void	intr_setup(int level, ih_func_t *ihf, int pri, iv_func_t *ivf,
 	    void *iva);
 void	intr_init1(void);

sys/sparc64/pci/psycho.c

@@ -87,7 +87,8 @@ static int psycho_find_intrmap(struct psycho_softc *, u_int, bus_addr_t *,
 static driver_filter_t psycho_dmasync;
 static void psycho_intr_enable(void *);
 static void psycho_intr_disable(void *);
-static void psycho_intr_eoi(void *);
+static void psycho_intr_assign(void *);
+static void psycho_intr_clear(void *);
 static bus_space_tag_t psycho_alloc_bus_tag(struct psycho_softc *, int);
 
 /* Interrupt handlers */
@@ -169,7 +170,8 @@ static SLIST_HEAD(, psycho_softc) psycho_softcs =
 static const struct intr_controller psycho_ic = {
 	psycho_intr_enable,
 	psycho_intr_disable,
-	psycho_intr_eoi
+	psycho_intr_assign,
+	psycho_intr_clear
 };
 
 struct psycho_icarg {
@@ -1099,7 +1101,17 @@ psycho_intr_disable(void *arg)
 }
 
 static void
-psycho_intr_eoi(void *arg)
+psycho_intr_assign(void *arg)
+{
+	struct intr_vector *iv = arg;
+	struct psycho_icarg *pica = iv->iv_icarg;
+
+	PSYCHO_WRITE8(pica->pica_sc, pica->pica_map, INTMAP_TID(
+	    PSYCHO_READ8(pica->pica_sc, pica->pica_map), iv->iv_mid));
+}
+
+static void
+psycho_intr_clear(void *arg)
 {
 	struct intr_vector *iv = arg;
 	struct psycho_icarg *pica = iv->iv_icarg;

sys/sparc64/sbus/sbus.c

@@ -196,7 +196,8 @@ static struct sbus_devinfo * sbus_setup_dinfo(device_t, struct sbus_softc *,
 static void sbus_destroy_dinfo(struct sbus_devinfo *);
 static void sbus_intr_enable(void *);
 static void sbus_intr_disable(void *);
-static void sbus_intr_eoi(void *);
+static void sbus_intr_assign(void *);
+static void sbus_intr_clear(void *);
 static int sbus_find_intrmap(struct sbus_softc *, u_int, bus_addr_t *,
     bus_addr_t *);
 static bus_space_tag_t sbus_alloc_bustag(struct sbus_softc *);
@@ -253,7 +254,8 @@ DRIVER_MODULE(sbus, nexus, sbus_driver, sbus_devclass, 0, 0);
 static const struct intr_controller sbus_ic = {
 	sbus_intr_enable,
 	sbus_intr_disable,
-	sbus_intr_eoi
+	sbus_intr_assign,
+	sbus_intr_clear
 };
 
 struct sbus_icarg {
@@ -667,6 +669,7 @@ sbus_intr_enable(void *arg)
 	SYSIO_WRITE8(sica->sica_sc, sica->sica_map,
 	    INTMAP_ENABLE(iv->iv_vec, iv->iv_mid));
 }
+
 static void
 sbus_intr_disable(void *arg)
 {
@@ -677,7 +680,17 @@ sbus_intr_disable(void *arg)
 }
 
 static void
-sbus_intr_eoi(void *arg)
+sbus_intr_assign(void *arg)
+{
+	struct intr_vector *iv = arg;
+	struct sbus_icarg *sica = iv->iv_icarg;
+
+	SYSIO_WRITE8(sica->sica_sc, sica->sica_map, INTMAP_TID(
+	    SYSIO_READ8(sica->sica_sc, sica->sica_map), iv->iv_mid));
+}
+
+static void
+sbus_intr_clear(void *arg)
 {
 	struct intr_vector *iv = arg;
 	struct sbus_icarg *sica = iv->iv_icarg;

sys/sparc64/sparc64/intr_machdep.c

@@ -63,16 +63,16 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
-#include <sys/queue.h>
 #include <sys/bus.h>
 #include <sys/errno.h>
 #include <sys/interrupt.h>
-#include <sys/ktr.h>
+#include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/pcpu.h>
 #include <sys/proc.h>
-#include <sys/vmmeter.h>
+#include <sys/smp.h>
+#include <sys/sx.h>
 
 #include <machine/frame.h>
 #include <machine/intr_machdep.h>
@@ -102,24 +102,30 @@ static const char *pil_names[] = {
 };
 
 /* protect the intr_vectors table */
-static struct mtx intr_table_lock;
+static struct sx intr_table_lock;
+/* protect intrcnt_index */
+static struct mtx intrcnt_lock;
 
-static void intr_enable_eoi(void *);
+#ifdef SMP
+static int assign_cpu;
+
+static void intr_assign_next_cpu(struct intr_vector *iv);
+#endif
+
+static int intr_assign_cpu(void *arg, u_char cpu);
 static void intr_execute_handlers(void *);
 static void intr_stray_level(struct trapframe *);
 static void intr_stray_vector(void *);
 static int intrcnt_setname(const char *, int);
 static void intrcnt_updatename(int, const char *, int);
 
-/*
- * not MPSAFE
- */
 static void
 intrcnt_updatename(int vec, const char *name, int ispil)
 {
 	static int intrcnt_index, stray_pil_index, stray_vec_index;
 	int name_index;
 
+	mtx_lock_spin(&intrcnt_lock);
 	if (intrnames[0] == '\0') {
 		/* for bitbucket */
 		if (bootverbose)
@@ -154,6 +160,7 @@ intrcnt_updatename(int vec, const char *name, int ispil)
 		intr_countp[vec] = name_index;
 	else
 		pil_countp[vec] = name_index;
+	mtx_unlock_spin(&intrcnt_lock);
 }
 
 static int
@@ -171,19 +178,19 @@ void
 intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva)
 {
 	char pilname[MAXCOMLEN + 1];
-	u_long ps;
+	register_t s;
 
-	ps = intr_disable();
+	s = intr_disable();
 	if (vec != -1) {
 		intr_vectors[vec].iv_func = ivf;
 		intr_vectors[vec].iv_arg = iva;
 		intr_vectors[vec].iv_pri = pri;
 		intr_vectors[vec].iv_vec = vec;
 	}
+	intr_handlers[pri] = ihf;
+	intr_restore(s);
 	snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]);
 	intrcnt_updatename(pri, pilname, 1);
-	intr_handlers[pri] = ihf;
-	intr_restore(ps);
 }
 
 static void
@@ -230,19 +237,35 @@ void
 intr_init2()
 {
 
-	mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN);
+	sx_init(&intr_table_lock, "intr sources");
+	mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
 }
 
-static void
-intr_enable_eoi(void *arg)
+static int
+intr_assign_cpu(void *arg, u_char cpu)
 {
+#ifdef SMP
+	struct pcpu *pc;
 	struct intr_vector *iv;
-	const struct intr_controller *ic;
 
-	iv = arg;
-	ic = iv->iv_ic;
-	ic->ic_enable(iv);
-	ic->ic_eoi(iv);
+	/*
+	 * Don't do anything during early boot.  We will pick up the
+	 * assignment once the APs are started.
+	 */
+	if (assign_cpu && cpu != NOCPU) {
+		pc = pcpu_find(cpu);
+		if (pc == NULL)
+			return (EINVAL);
+		iv = arg;
+		sx_xlock(&intr_table_lock);
+		iv->iv_mid = pc->pc_mid;
+		iv->iv_ic->ic_assign(iv);
+		sx_xunlock(&intr_table_lock);
+	}
+	return (0);
+#else
+	return (EOPNOTSUPP);
+#endif
 }
 
 static void
@@ -263,32 +286,21 @@ intr_controller_register(int vec, const struct intr_controller *ic,
 	struct intr_vector *iv;
 	int error;
 
+	if (vec < 0 || vec >= IV_MAX)
+		return (EINVAL);
+	sx_xlock(&intr_table_lock);
 	iv = &intr_vectors[vec];
-	mtx_lock_spin(&intr_table_lock);
 	ie = iv->iv_event;
-	mtx_unlock_spin(&intr_table_lock);
+	sx_xunlock(&intr_table_lock);
 	if (ie != NULL)
 		return (EEXIST);
-	/*
-	 * Testing shows that at least with the interrupt controllers of
-	 * Psycho and Schizo bridges enabling an interrupt doesn't cause
-	 * an outstanding interrupt to be issued to the CPU. Thus we can't
-	 * use a function doing disable+EOI for the "disable" pointer as
-	 * done on other architectures because this would lead to a lost
-	 * interrupt if it triggers while we are still processing the
-	 * previous one. Instead we use an enable+EOI approach because as
-	 * outlined in the Tomatillo documentation clearing an interrupt
-	 * in the interrupt controller causes it to be (re)issued to the
-	 * CPU as long as the source of a level sensitive interrupt is
-	 * not cleared.
-	 */
-	error = intr_event_create(&ie, iv, 0, vec, NULL, intr_enable_eoi,
-	    intr_enable_eoi, NULL, "vec%d:", vec);
+	error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear,
+	    ic->ic_clear, intr_assign_cpu, "vec%d:", vec);
 	if (error != 0)
 		return (error);
-	mtx_lock_spin(&intr_table_lock);
+	sx_xlock(&intr_table_lock);
 	if (iv->iv_event != NULL) {
-		mtx_unlock_spin(&intr_table_lock);
+		sx_xunlock(&intr_table_lock);
 		intr_event_destroy(ie);
 		return (EEXIST);
 	}
@@ -296,7 +308,7 @@ intr_controller_register(int vec, const struct intr_controller *ic,
 	iv->iv_icarg = icarg;
 	iv->iv_event = ie;
 	iv->iv_mid = PCPU_GET(mid);
-	mtx_unlock_spin(&intr_table_lock);
+	sx_xunlock(&intr_table_lock);
 	return (0);
 }
 
@@ -310,20 +322,20 @@ inthand_add(const char *name, int vec, driver_filter_t *filt,
 	struct intr_vector *iv;
 	int error, fast;
 
+	if (vec < 0 || vec >= IV_MAX)
+		return (EINVAL);
+	sx_xlock(&intr_table_lock);
 	iv = &intr_vectors[vec];
-	mtx_lock_spin(&intr_table_lock);
 	ic = iv->iv_ic;
 	ie = iv->iv_event;
-	mtx_unlock_spin(&intr_table_lock);
+	sx_xunlock(&intr_table_lock);
 	if (ic == NULL || ie == NULL)
 		return (EINVAL);
-
 	error = intr_event_add_handler(ie, name, filt, handler, arg,
 	    intr_priority(flags), flags, cookiep);
 	if (error != 0)
 		return (error);
-
-	mtx_lock_spin(&intr_table_lock);
+	sx_xlock(&intr_table_lock);
 	/* Disable the interrupt while we fiddle with it. */
 	ic->ic_disable(iv);
 	iv->iv_refcnt++;
@@ -350,9 +362,14 @@ inthand_add(const char *name, int vec, driver_filter_t *filt,
 	}
 	intr_stray_count[vec] = 0;
 	intrcnt_updatename(vec, ie->ie_fullname, 0);
+#ifdef SMP
+	if (assign_cpu)
+		intr_assign_next_cpu(iv);
+#endif
+	ic->ic_enable(iv);
 	/* Ensure the interrupt is cleared, it might have triggered before. */
-	intr_enable_eoi(iv);
-	mtx_unlock_spin(&intr_table_lock);
+	ic->ic_clear(iv);
+	sx_xunlock(&intr_table_lock);
 	return (0);
 }
 
@@ -362,14 +379,16 @@ inthand_remove(int vec, void *cookie)
 	struct intr_vector *iv;
 	int error;
 
+	if (vec < 0 || vec >= IV_MAX)
+		return (EINVAL);
 	error = intr_event_remove_handler(cookie);
 	if (error == 0) {
 		/*
 		 * XXX: maybe this should be done regardless of whether
 		 * intr_event_remove_handler() succeeded?
 		 */
+		sx_xlock(&intr_table_lock);
 		iv = &intr_vectors[vec];
-		mtx_lock_spin(&intr_table_lock);
 		iv->iv_refcnt--;
 		if (iv->iv_refcnt == 0) {
 			/*
@@ -379,7 +398,109 @@ inthand_remove(int vec, void *cookie)
 			intr_setup(PIL_LOW, intr_fast, vec,
 			    intr_stray_vector, iv);
 		}
-		mtx_unlock_spin(&intr_table_lock);
+		sx_xunlock(&intr_table_lock);
 	}
 	return (error);
 }
+
+#ifdef SMP
+/*
+ * Support for balancing interrupt sources across CPUs.  For now we just
+ * allocate CPUs round-robin.
+ */
+
+/* The BSP is always a valid target. */
+static cpumask_t intr_cpus = (1 << 0);
+static int current_cpu;
+
+static void
+intr_assign_next_cpu(struct intr_vector *iv)
+{
+	struct pcpu *pc;
+
+	sx_assert(&intr_table_lock, SA_XLOCKED);
+
+	/*
+	 * Assign this source to a CPU in a round-robin fashion.
+	 */
+	pc = pcpu_find(current_cpu);
+	if (pc == NULL)
+		return;
+	iv->iv_mid = pc->pc_mid;
+	iv->iv_ic->ic_assign(iv);
+	do {
+		current_cpu++;
+		if (current_cpu > mp_maxid)
+			current_cpu = 0;
+	} while (!(intr_cpus & (1 << current_cpu)));
+}
+
+/* Attempt to bind the specified IRQ to the specified CPU. */
+int
+intr_bind(int vec, u_char cpu)
+{
+	struct intr_vector *iv;
+
+	if (vec < 0 || vec >= IV_MAX)
+		return (EINVAL);
+	iv = &intr_vectors[vec];
+	if (iv == NULL)
+		return (EINVAL);
+	return (intr_event_bind(iv->iv_event, cpu));
+}
+
+/*
+ * Add a CPU to our mask of valid CPUs that can be destinations of
+ * interrupts.
+ */
+void
+intr_add_cpu(u_int cpu)
+{
+
+	if (cpu >= MAXCPU)
+		panic("%s: Invalid CPU ID", __func__);
+	if (bootverbose)
+		printf("INTR: Adding CPU %d as a target\n", cpu);
+
+	intr_cpus |= (1 << cpu);
+}
+
+/*
+ * Distribute all the interrupt sources among the available CPUs once the
+ * AP's have been launched.
+ */
+static void
+intr_shuffle_irqs(void *arg __unused)
+{
+	struct pcpu *pc;
+	struct intr_vector *iv;
+	int i;
+
+	/* Don't bother on UP. */
+	if (mp_ncpus == 1)
+		return;
+
+	/* Round-robin assign a CPU to each enabled source. */
+	sx_xlock(&intr_table_lock);
+	assign_cpu = 1;
+	for (i = 0; i < IV_MAX; i++) {
+		iv = &intr_vectors[i];
+		if (iv != NULL && iv->iv_refcnt > 0) {
+			/*
+			 * If this event is already bound to a CPU,
+			 * then assign the source to that CPU instead
+			 * of picking one via round-robin.
+			 */
+			if (iv->iv_event->ie_cpu != NOCPU &&
+			    (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) {
+				iv->iv_mid = pc->pc_mid;
+				iv->iv_ic->ic_assign(iv);
+			} else
+				intr_assign_next_cpu(iv);
+		}
+	}
+	sx_xunlock(&intr_table_lock);
+}
+SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs,
+    NULL);
+#endif

sys/sparc64/sparc64/mp_machdep.c

@@ -171,7 +171,7 @@ cpu_mp_setmaxid(void)
 {
 	char buf[128];
 	phandle_t child;
-	int cpus;
+	u_int cpus;
 
 	all_cpus = 1 << curcpu;
 	mp_ncpus = 1;
@@ -251,7 +251,7 @@ cpu_mp_start(void)
 	phandle_t child;
 	u_int clock;
 	u_int mid;
-	int cpuid;
+	u_int cpuid;
 
 	mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN);
 
@@ -302,6 +302,7 @@ cpu_mp_start(void)
 		pc->pc_node = child;
 
 		all_cpus |= 1 << cpuid;
+		intr_add_cpu(cpuid);
 	}
 	KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS,
 	    ("%s: can only IPI a maximum of %d JBus-CPUs",

sys/sparc64/sparc64/nexus.c

@@ -93,6 +93,9 @@ static bus_get_resource_list_t nexus_get_resource_list;
 static bus_get_dma_tag_t nexus_get_dma_tag;
 static ofw_bus_get_devinfo_t nexus_get_devinfo;
 
+#ifdef SMP
+static int nexus_bind_intr(device_t, device_t, struct resource *, int);
+#endif
 static int nexus_inlist(const char *, const char **);
 static struct nexus_devinfo * nexus_setup_dinfo(device_t, phandle_t);
 static void nexus_destroy_dinfo(struct nexus_devinfo *);
@@ -119,6 +122,9 @@ static device_method_t nexus_methods[] = {
 	DEVMETHOD(bus_release_resource,	nexus_release_resource),
 	DEVMETHOD(bus_setup_intr,	nexus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	nexus_teardown_intr),
+#ifdef SMP
+	DEVMETHOD(bus_bind_intr,	nexus_bind_intr),
+#endif
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
 	DEVMETHOD(bus_get_resource_list, nexus_get_resource_list),
 	DEVMETHOD(bus_get_dma_tag,	nexus_get_dma_tag),
@@ -311,6 +317,15 @@ nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
 	return (0);
 }
 
+#ifdef SMP
+static int
+nexus_bind_intr(device_t dev, device_t child, struct resource *r, int cpu)
+{
+
+	return (intr_bind(rman_get_start(r), cpu));
+}
+#endif
+
 static struct resource *
 nexus_alloc_resource(device_t bus, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)

sys/sparc64/sparc64/upa.c

@@ -108,6 +108,7 @@ static ofw_bus_get_devinfo_t upa_get_devinfo;
 
 static void upa_intr_enable(void *);
 static void upa_intr_disable(void *);
+static void upa_intr_assign(void *);
 static struct upa_devinfo *upa_setup_dinfo(device_t, struct upa_softc *,
     phandle_t, uint32_t);
 static void upa_destroy_dinfo(struct upa_devinfo *);
@@ -154,6 +155,7 @@ DRIVER_MODULE(upa, nexus, upa_driver, upa_devclass, 0, 0);
 static const struct intr_controller upa_ic = {
 	upa_intr_enable,
 	upa_intr_disable,
+	upa_intr_assign,
 	/* The interrupts are pulse type and thus automatically cleared. */
 	NULL
 };
@@ -469,6 +471,17 @@ upa_intr_disable(void *arg)
 	(void)UPA_READ(uica->uica_sc, uica->uica_imr, 0x0);
 }
 
+static void
+upa_intr_assign(void *arg)
+{
+	struct intr_vector *iv = arg;
+	struct upa_icarg *uica = iv->iv_icarg;
+
+	UPA_WRITE(uica->uica_sc, uica->uica_imr, 0x0, INTMAP_TID(
+	    UPA_READ(uica->uica_sc, uica->uica_imr, 0x0), iv->iv_mid));
+	(void)UPA_READ(uica->uica_sc, uica->uica_imr, 0x0);
+}
+
 static int
 upa_setup_intr(device_t dev, device_t child, struct resource *ires, int flags,
     driver_filter_t *filt, driver_intr_t *func, void *arg, void **cookiep)