--- //depot/projects/smpng/sys/amd64/amd64/intr_machdep.c	2008/12/04 20:58:27
+++ //depot/user/jhb/msi/amd64/amd64/intr_machdep.c	2008/12/04 22:14:32
@@ -442,7 +442,8 @@
 	/*
 	 * Assign this source to a local APIC in a round-robin fashion.
 	 */
-	isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+	if (isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]) != 0)
+		return;
 	do {
 		current_cpu++;
 		if (current_cpu > mp_maxid)
--- //depot/projects/smpng/sys/amd64/amd64/io_apic.c	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/amd64/amd64/io_apic.c	2009/02/04 22:38:20
@@ -120,7 +120,7 @@
 static int	ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 		    enum intr_polarity pol);
 static void	ioapic_resume(struct pic *pic);
-static void	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
 static void	ioapic_program_intpin(struct ioapic_intsrc *intpin);
 
 static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
@@ -322,7 +322,7 @@
 	mtx_unlock_spin(&icu_lock);
 }
 
-static void
+static int
 ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
@@ -342,7 +342,7 @@
 	old_vector = intpin->io_vector;
 	old_id = intpin->io_cpu;
 	if (old_vector && apic_id == old_id)
-		return;
+		return (0);
 
 	/*
 	 * Allocate an APIC vector for this interrupt pin.  Once
@@ -364,6 +364,7 @@
 	 */
 	if (old_vector)
 		apic_free_vector(old_id, old_vector, intpin->io_irq);
+	return (0);
 }
 
 static void
--- //depot/projects/smpng/sys/amd64/amd64/msi.c	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/amd64/amd64/msi.c	2009/02/04 22:38:20
@@ -125,7 +125,7 @@
 static int	msi_source_pending(struct intsrc *isrc);
 static int	msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 		    enum intr_polarity pol);
-static void	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
 
 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
 		       msi_enable_intr, msi_disable_intr, msi_vector,
@@ -197,7 +197,7 @@
 	return (ENODEV);
 }
 
-static void
+static int
 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
@@ -205,15 +205,23 @@
 	u_int old_id;
 	int vector;
 
+	/*
+	 * Only allow CPUs to be assigned to the first message for an
+	 * MSI group.
+	 */
+	if (msi->msi_msix == 0 && msi->msi_first != msi)
+		return (EINVAL);
+
 	/* Store information to free existing irq. */
 	old_vector = msi->msi_vector;
 	old_id = msi->msi_cpu;
 	if (old_vector && old_id == apic_id)
-		return;
+		return (0);
+
 	/* Allocate IDT vector on this cpu. */
 	vector = apic_alloc_vector(apic_id, msi->msi_irq);
 	if (vector == 0)
-		return; /* XXX alloc_vector panics on failure. */
+		return (EIO); /* XXX alloc_vector panics on failure. */
 	msi->msi_cpu = apic_id;
 	msi->msi_vector = vector;
 	if (bootverbose)
@@ -227,6 +235,7 @@
 	 */
 	if (old_vector)
 		apic_free_vector(old_id, old_vector, msi->msi_irq);
+	return (0);
 }
 
 
@@ -441,6 +450,9 @@
 
 	*addr = INTEL_ADDR(msi);
 	*data = INTEL_DATA(msi);
+	/* DBG */
+	printf("%s: mapped IRQ %d to address %#lx, data %#x\n", __func__, irq,
+	    (u_long)*addr, (u_int)*data);
 	mtx_unlock(&msi_lock);
 	return (0);
 }
--- //depot/projects/smpng/sys/amd64/include/intr_machdep.h	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/amd64/include/intr_machdep.h	2009/02/04 22:38:20
@@ -93,7 +93,7 @@
 	void (*pic_resume)(struct pic *);
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
-	void (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
+	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
 	STAILQ_ENTRY(pic) pics;
 };
 
--- //depot/projects/smpng/sys/amd64/include/reg.h	2009/04/07 17:48:51
+++ //depot/user/jhb/msi/amd64/include/reg.h	2009/04/07 19:19:09
@@ -105,7 +105,7 @@
 #define	DBREG_DR7_EXEC		0x00	/* break on execute       */
 #define	DBREG_DR7_WRONLY	0x01	/* break on write         */
 #define	DBREG_DR7_RDWR		0x03	/* break on read or write */
-#define	DBREG_DR7_MASK(i)	((u_long)0xf << ((i) * 4 + 16) | 0x3 << (i) * 2)
+#define	DBREG_DR7_MASK(i)	(0xful << ((i) * 4 + 16) | 0x3 << (i) * 2)
 #define	DBREG_DR7_SET(i, len, access, enable)				\
 	((u_long)((len) << 2 | (access)) << ((i) * 4 + 16) | (enable) << (i) * 2)
 #define	DBREG_DR7_GD		0x2000
--- //depot/projects/smpng/sys/amd64/isa/atpic.c	2008/03/24 19:59:34
+++ //depot/user/jhb/msi/amd64/isa/atpic.c	2008/03/24 21:46:53
@@ -143,7 +143,7 @@
 static int atpic_source_pending(struct intsrc *isrc);
 static int atpic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
     enum intr_polarity pol);
-static void atpic_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int atpic_assign_cpu(struct intsrc *isrc, u_int apic_id);
 static void i8259_init(struct atpic *pic, int slave);
 
 static struct atpic atpics[] = {
@@ -357,7 +357,7 @@
 	return (0);
 }
 
-static void
+static int
 atpic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 
--- //depot/projects/smpng/sys/dev/pci/pci.c	2009/04/14 19:06:19
+++ //depot/user/jhb/msi/dev/pci/pci.c	2009/04/14 19:48:26
@@ -1099,6 +1099,10 @@
 
 	/* Enable MSI -> HT mapping. */
 	pci_ht_map_msi(dev, address);
+
+	/* DBG */
+	device_printf(dev, "Enabling MSI-X (%d): addr %#jx, data %#x\n",
+	    index, (uintmax_t)address, (u_int)data);
 }
 
 void
@@ -1115,6 +1119,8 @@
 		val |= PCIM_MSIX_VCTRL_MASK;
 		bus_write_4(msix->msix_table_res, offset, val);
 	}
+	/* DBG */
+	device_printf(dev, "Masking MSI-X (%d)\n", index);
 }
 
 void
@@ -1131,6 +1137,8 @@
 		val &= ~PCIM_MSIX_VCTRL_MASK;
 		bus_write_4(msix->msix_table_res, offset, val);
 	}
+	/* DBG */
+	device_printf(dev, "Unmasking MSI-X (%d)\n", index);
 }
 
 int
@@ -1593,6 +1601,10 @@
 
 	/* Enable MSI -> HT mapping. */
 	pci_ht_map_msi(dev, address);
+
+	/* DBG */
+	device_printf(dev, "Enabling MSI: addr %#jx, data %#x\n",
+	    (uintmax_t)address, (u_int)data);
 }
 
 void
@@ -1608,6 +1620,8 @@
 	msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
 	pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
 	    2);
+	/* DBG */
+	device_printf(dev, "Disabling MSI\n");
 }
 
 /*
--- //depot/projects/smpng/sys/dev/pci/vga_pci.c	2009/03/10 21:55:15
+++ //depot/user/jhb/msi/dev/pci/vga_pci.c	2009/03/11 12:28:19
@@ -211,7 +211,7 @@
 }
 
 static void
-vga_pci_write_config(device_t dev, device_t child, int reg, 
+vga_pci_write_config(device_t dev, device_t child, int reg,
     uint32_t val, int width)
 {
 
@@ -388,7 +388,6 @@
 	DEVMETHOD(bus_write_ivar,	vga_pci_write_ivar),
 	DEVMETHOD(bus_setup_intr,	vga_pci_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	vga_pci_teardown_intr),
-
 	DEVMETHOD(bus_alloc_resource,	vga_pci_alloc_resource),
 	DEVMETHOD(bus_release_resource,	vga_pci_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
--- //depot/projects/smpng/sys/i386/i386/intr_machdep.c	2008/12/04 20:58:27
+++ //depot/user/jhb/msi/i386/i386/intr_machdep.c	2008/12/04 22:14:32
@@ -408,7 +408,8 @@
 	/*
 	 * Assign this source to a local APIC in a round-robin fashion.
 	 */
-	isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+	if (isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]) != 0)
+		return;
 	do {
 		current_cpu++;
 		if (current_cpu > mp_maxid)
--- //depot/projects/smpng/sys/i386/i386/io_apic.c	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/i386/i386/io_apic.c	2009/02/04 22:38:20
@@ -120,7 +120,7 @@
 static int	ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 		    enum intr_polarity pol);
 static void	ioapic_resume(struct pic *pic);
-static void	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
 static void	ioapic_program_intpin(struct ioapic_intsrc *intpin);
 
 static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
@@ -322,7 +322,7 @@
 	mtx_unlock_spin(&icu_lock);
 }
 
-static void
+static int
 ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
@@ -342,7 +342,7 @@
 	old_vector = intpin->io_vector;
 	old_id = intpin->io_cpu;
 	if (old_vector && apic_id == old_id)
-		return;
+		return (0);
 
 	/*
 	 * Allocate an APIC vector for this interrupt pin.  Once
@@ -364,6 +364,7 @@
 	 */
 	if (old_vector)
 		apic_free_vector(old_id, old_vector, intpin->io_irq);
+	return (0);
 }
 
 static void
--- //depot/projects/smpng/sys/i386/i386/msi.c	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/i386/i386/msi.c	2009/02/04 22:38:20
@@ -125,7 +125,7 @@
 static int	msi_source_pending(struct intsrc *isrc);
 static int	msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 		    enum intr_polarity pol);
-static void	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
 
 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
 		       msi_enable_intr, msi_disable_intr, msi_vector,
@@ -197,7 +197,7 @@
 	return (ENODEV);
 }
 
-static void
+static int
 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
@@ -205,15 +205,23 @@
 	u_int old_id;
 	int vector;
 
+	/*
+	 * Only allow CPUs to be assigned to the first message for an
+	 * MSI group.
+	 */
+	if (msi->msi_msix == 0 && msi->msi_first != msi)
+		return (EINVAL);
+
 	/* Store information to free existing irq. */
 	old_vector = msi->msi_vector;
 	old_id = msi->msi_cpu;
 	if (old_vector && old_id == apic_id)
-		return;
+		return (0);
+
 	/* Allocate IDT vector on this cpu. */
 	vector = apic_alloc_vector(apic_id, msi->msi_irq);
 	if (vector == 0)
-		return; /* XXX alloc_vector panics on failure. */
+		return (EIO); /* XXX alloc_vector panics on failure. */
 	msi->msi_cpu = apic_id;
 	msi->msi_vector = vector;
 	if (bootverbose)
@@ -227,6 +235,7 @@
 	 */
 	if (old_vector)
 		apic_free_vector(old_id, old_vector, msi->msi_irq);
+	return (0);
 }
 
 
--- //depot/projects/smpng/sys/i386/include/intr_machdep.h	2009/02/04 21:27:39
+++ //depot/user/jhb/msi/i386/include/intr_machdep.h	2009/02/04 22:38:20
@@ -93,7 +93,7 @@
 	void (*pic_resume)(struct pic *);
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
-	void (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
+	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
 	STAILQ_ENTRY(pic) pics;
 };
 
--- //depot/projects/smpng/sys/i386/isa/atpic.c	2008/03/24 19:59:34
+++ //depot/user/jhb/msi/i386/isa/atpic.c	2008/03/24 21:46:53
@@ -161,7 +161,7 @@
 static int atpic_source_pending(struct intsrc *isrc);
 static int atpic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
     enum intr_polarity pol);
-static void atpic_assign_cpu(struct intsrc *isrc, u_int apic_id);
+static int atpic_assign_cpu(struct intsrc *isrc, u_int apic_id);
 static void i8259_init(struct atpic *pic, int slave);
 
 static struct atpic atpics[] = {
@@ -389,7 +389,7 @@
 #endif /* PC98 */
 }
 
-static void
+static int
 atpic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 
--- //depot/projects/smpng/sys/notes	2009/02/18 22:05:55
+++ //depot/user/jhb/msi/notes	2009/02/20 16:15:34
@@ -73,3 +73,14 @@
 - jhb_socket - socket hacking
 
 Space reserved for child branches:
+- HT
+  - need to write address window register if it's not correct
+- quirks
+  + use pci_quirks table
+  + blacklist chipsets based on 0,0,0 devid
+  + blacklist devices behind bad pci-pci bridges
+  - there are too many old chipsets to quirk, let's take a different approach
+    + default to enabling MSI on systems with a 0,0,0 devid that supports
+      PCI-express and use existing ability to blacklist broken PCI-e chipsets
+    - default to not enabling MSI on other systems but add explicit
+      whitelisting for any non-PCI-e chipset that does work