From 64258f4644607a8d4a142a3cab570d18141a50f1 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Fri, 5 Aug 2016 17:13:25 +0000 Subject: [PATCH] MFC 302181,302635: Disable MSI-X migration on older Xen hypervisors. 302181: Add a tunable to disable migration of MSI-X interrupts. The new 'machdep.disable_msix_migration' tunable can be set to 1 to disable migration of MSI-X interrupts. Xen versions prior to 4.6.0 do not properly handle updates to MSI-X table entries after the initial write. In particular, the operation to unmask a table entry after updating it during migration is not propagated to the "real" table for passthrough devices causing the interrupt to remain masked. At least some systems in EC2 are affected by this bug when using SRIOV. The tunable can be set in loader.conf as a workaround. 302635: xen: automatically disable MSI-X interrupt migration If the hypervisor version is smaller than 4.6.0. Xen commits 74fd00 and 70a3cb are required on the hypervisor side for this to be fixed, and those are only included in 4.6.0, so stay on the safe side and disable MSI-X interrupt migration on anything older than 4.6.0. It should not cause major performance degradation unless a lot of MSI-X interrupts are allocated. --- sys/amd64/include/intr_machdep.h | 2 ++ sys/i386/include/intr_machdep.h | 2 ++ sys/x86/x86/msi.c | 23 +++++++++++++++++++++++ sys/x86/xen/hvm.c | 24 ++++++++++++++++++++++-- 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h index fb71b5a5e862..7626c81523fc 100644 --- a/sys/amd64/include/intr_machdep.h +++ b/sys/amd64/include/intr_machdep.h @@ -144,6 +144,8 @@ struct nmi_pcpu { extern struct mtx icu_lock; extern int elcr_found; +extern int msix_disable_migration; + #ifndef DEV_ATPIC void atpic_reset(void); #endif diff --git a/sys/i386/include/intr_machdep.h b/sys/i386/include/intr_machdep.h index 8fb61a54bc69..b71bbb64a0e6 100644 --- a/sys/i386/include/intr_machdep.h +++ b/sys/i386/include/intr_machdep.h @@ -141,6 +141,8 @@ struct trapframe; extern struct mtx icu_lock; extern int elcr_found; +extern int msix_disable_migration; + #ifndef DEV_ATPIC void atpic_reset(void); #endif diff --git a/sys/x86/x86/msi.c b/sys/x86/x86/msi.c index 381f09760719..37f0e1662b13 100644 --- a/sys/x86/x86/msi.c +++ b/sys/x86/x86/msi.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -134,6 +135,20 @@ struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source, msi_source_pending, NULL, NULL, msi_config_intr, msi_assign_cpu }; +/** + * Xen hypervisors prior to 4.6.0 do not properly handle updates to + * enabled MSI-X table entries. Allow migration of MSI-X interrupts + * to be disabled via a tunable. Values have the following meaning: + * + * -1: automatic detection by FreeBSD + * 0: enable migration + * 1: disable migration + */ +int msix_disable_migration = -1; +SYSCTL_INT(_machdep, OID_AUTO, disable_msix_migration, CTLFLAG_RDTUN, + &msix_disable_migration, 0, + "Disable migration of MSI-X interrupts between CPUs"); + static int msi_enabled; static int msi_last_irq; static struct mtx msi_lock; @@ -212,6 +227,9 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id) if (msi->msi_first != msi) return (EINVAL); + if (msix_disable_migration && msi->msi_msix) + return (EINVAL); + /* Store information to free existing irq. */ old_vector = msi->msi_vector; old_id = msi->msi_cpu; @@ -284,6 +302,11 @@ msi_init(void) return; } + if (msix_disable_migration == -1) { + /* The default is to allow migration of MSI-X interrupts. */ + msix_disable_migration = 0; + } + msi_enabled = 1; intr_register_pic(&msi_pic); mtx_init(&msi_lock, "msi", NULL, MTX_DEF); diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index 6c6f15305228..68f7fde26c4a 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -385,9 +385,29 @@ xen_hvm_init_hypercall_stubs(void) return (ENXIO); if (hypercall_stubs == NULL) { + int major, minor; + do_cpuid(base + 1, regs); - printf("XEN: Hypervisor version %d.%d detected.\n", - regs[0] >> 16, regs[0] & 0xffff); + + major = regs[0] >> 16; + minor = regs[0] & 0xffff; + printf("XEN: Hypervisor version %d.%d detected.\n", major, + minor); + + if (((major < 4) || (major == 4 && minor <= 5)) && + msix_disable_migration == -1) { + /* + * Xen hypervisors prior to 4.6.0 do not properly + * handle updates to enabled MSI-X table entries, + * so disable MSI-X interrupt migration in that + * case. + */ + if (bootverbose) + printf( +"Disabling MSI-X interrupt migration due to Xen hypervisor bug.\n" +"Set machdep.msix_disable_migration=0 to forcefully enable it.\n"); + msix_disable_migration = 1; + } } /*