--- //depot/projects/smpng/sys/i386/i386/io_apic.c 2005/04/14 18:55:16 +++ //depot/user/jhb/intr/i386/i386/io_apic.c 2005/04/26 21:22:01 @@ -54,10 +54,10 @@ #define IOAPIC_REDTBL_LO(i) (IOAPIC_REDTBL + (i) * 2) #define IOAPIC_REDTBL_HI(i) (IOAPIC_REDTBL_LO(i) + 1) -#define VECTOR_EXTINT 252 -#define VECTOR_NMI 253 -#define VECTOR_SMI 254 -#define VECTOR_DISABLED 255 +#define IRQ_EXTINT 252 +#define IRQ_NMI 253 +#define IRQ_SMI 254 +#define IRQ_DISABLED 255 #define DEST_NONE -1 @@ -83,6 +83,7 @@ struct ioapic_intsrc { struct intsrc io_intsrc; u_int io_intpin:8; + u_int io_irq:8; u_int io_vector:8; u_int io_activehi:1; u_int io_edgetrigger:1; @@ -105,7 +106,7 @@ static u_int ioapic_read(volatile ioapic_t *apic, int reg); static void ioapic_write(volatile ioapic_t *apic, int reg, u_int val); static const char *ioapic_bus_string(int bus_type); -static void ioapic_print_vector(struct ioapic_intsrc *intpin); +static void ioapic_print_irq(struct ioapic_intsrc *intpin); static void ioapic_enable_source(struct intsrc *isrc); static void ioapic_disable_source(struct intsrc *isrc, int eoi); static void ioapic_eoi_source(struct intsrc *isrc); @@ -170,25 +171,25 @@ } static void -ioapic_print_vector(struct ioapic_intsrc *intpin) +ioapic_print_irq(struct ioapic_intsrc *intpin) { - switch (intpin->io_vector) { - case VECTOR_DISABLED: + switch (intpin->io_irq) { + case IRQ_DISABLED: printf("disabled"); break; - case VECTOR_EXTINT: + case IRQ_EXTINT: printf("ExtINT"); break; - case VECTOR_NMI: + case IRQ_NMI: printf("NMI"); break; - case VECTOR_SMI: + case IRQ_SMI: printf("SMI"); break; default: printf("%s IRQ %u", ioapic_bus_string(intpin->io_bus), - intpin->io_vector); + intpin->io_irq); } } @@ -251,14 +252,20 @@ struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; uint32_t low, high, value; - /* For disabled pins, just ensure that they are masked. */ - if (intpin->io_vector == VECTOR_DISABLED) { + /* + * If a pin is completely invalid or if it is valid but hasn't + * been enabled yet, just ensure that the pin is masked. + */ + if (intpin->io_irq == IRQ_DISABLED || (intpin->io_irq < NUM_IO_INTS && + intpin->io_vector == 0)) { + mtx_lock_spin(&icu_lock); low = ioapic_read(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin)); if ((low & IOART_INTMASK) == IOART_INTMCLR) ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low | IOART_INTMSET); + mtx_unlock_spin(&icu_lock); return; } @@ -283,24 +290,26 @@ low |= IOART_INTALO; if (intpin->io_masked) low |= IOART_INTMSET; - switch (intpin->io_vector) { - case VECTOR_EXTINT: + switch (intpin->io_irq) { + case IRQ_EXTINT: KASSERT(intpin->io_edgetrigger, ("EXTINT not edge triggered")); low |= IOART_DELEXINT; break; - case VECTOR_NMI: + case IRQ_NMI: KASSERT(intpin->io_edgetrigger, ("NMI not edge triggered")); low |= IOART_DELNMI; break; - case VECTOR_SMI: + case IRQ_SMI: KASSERT(intpin->io_edgetrigger, ("SMI not edge triggered")); low |= IOART_DELSMI; break; default: - low |= IOART_DELLOPRI | apic_irq_to_idt(intpin->io_vector); + KASSERT(intpin->io_vector != 0, ("No vector for IRQ %u", + intpin->io_irq)); + low |= IOART_DELLOPRI | intpin->io_vector; } /* Write the values to the APIC. */ @@ -326,7 +335,7 @@ if (bootverbose) { printf("ioapic%u: routing intpin %u (", io->io_id, intpin->io_intpin); - ioapic_print_vector(intpin); + ioapic_print_irq(intpin); printf(") to cluster %u\n", intpin->io_dest); } ioapic_program_intpin(intpin); @@ -357,10 +366,27 @@ ioapic_enable_intr(struct intsrc *isrc) { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + struct ioapic *io = (struct ioapic *)isrc->is_pic; if (intpin->io_dest == DEST_NONE) { + /* + * Allocate an APIC vector for this interrupt pin. Once + * we have a vector we program the interrupt pin. Note + * that after we have booted ioapic_assign_cluster() + * will program the interrupt pin again, but it doesn't + * hurt to do that and trying to avoid that adds needless + * complication. + */ + intpin->io_vector = apic_alloc_vector(intpin->io_irq); + if (bootverbose) { + printf("ioapic%u: routing intpin %u (", io->io_id, + intpin->io_intpin); + ioapic_print_irq(intpin); + printf(") to vector %u\n", intpin->io_vector); + } + ioapic_program_intpin(intpin); ioapic_assign_cluster(intpin); - lapic_enable_intr(intpin->io_vector); + apic_enable_vector(intpin->io_vector); } } @@ -370,7 +396,7 @@ struct ioapic_intsrc *pin; pin = (struct ioapic_intsrc *)isrc; - return (pin->io_vector); + return (pin->io_irq); } static int @@ -378,6 +404,8 @@ { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + if (intpin->io_vector == 0) + return 0; return (lapic_intr_pending(intpin->io_vector)); } @@ -515,16 +543,16 @@ for (i = 0, intpin = io->io_pins; i < numintr; i++, intpin++) { intpin->io_intsrc.is_pic = (struct pic *)io; intpin->io_intpin = i; - intpin->io_vector = intbase + i; + intpin->io_irq = intbase + i; /* * Assume that pin 0 on the first I/O APIC is an ExtINT pin. * Assume that pins 1-15 are ISA interrupts and that all * other pins are PCI interrupts. */ - if (intpin->io_vector == 0) + if (intpin->io_irq == 0) ioapic_set_extint(io, i); - else if (intpin->io_vector < IOAPIC_ISA_INTS) { + else if (intpin->io_irq < IOAPIC_ISA_INTS) { intpin->io_bus = APIC_BUS_ISA; intpin->io_activehi = 1; intpin->io_edgetrigger = 1; @@ -542,9 +570,9 @@ * logical IDs to CPU clusters when they are enabled. */ intpin->io_dest = DEST_NONE; - if (bootverbose && intpin->io_vector != VECTOR_DISABLED) { + if (bootverbose && intpin->io_irq != IRQ_DISABLED) { printf("ioapic%u: intpin %d -> ", io->io_id, i); - ioapic_print_vector(intpin); + ioapic_print_irq(intpin); printf(" (%s, %s)\n", intpin->io_edgetrigger ? "edge" : "level", intpin->io_activehi ? "high" : "low"); @@ -565,7 +593,7 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (-1); - return (io->io_pins[pin].io_vector); + return (io->io_pins[pin].io_irq); } int @@ -576,9 +604,9 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (EINVAL); - if (io->io_pins[pin].io_vector == VECTOR_DISABLED) + if (io->io_pins[pin].io_irq == IRQ_DISABLED) return (EINVAL); - io->io_pins[pin].io_vector = VECTOR_DISABLED; + io->io_pins[pin].io_irq = IRQ_DISABLED; if (bootverbose) printf("ioapic%u: intpin %d disabled\n", io->io_id, pin); return (0); @@ -592,9 +620,9 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr || vector < 0) return (EINVAL); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); - io->io_pins[pin].io_vector = vector; + io->io_pins[pin].io_irq = vector; if (bootverbose) printf("ioapic%u: Routing IRQ %d -> intpin %d\n", io->io_id, vector, pin); @@ -611,7 +639,7 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (EINVAL); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_bus = bus_type; if (bootverbose) @@ -628,12 +656,12 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (EINVAL); - if (io->io_pins[pin].io_vector == VECTOR_NMI) + if (io->io_pins[pin].io_irq == IRQ_NMI) return (0); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; - io->io_pins[pin].io_vector = VECTOR_NMI; + io->io_pins[pin].io_irq = IRQ_NMI; io->io_pins[pin].io_masked = 0; io->io_pins[pin].io_edgetrigger = 1; io->io_pins[pin].io_activehi = 1; @@ -651,12 +679,12 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (EINVAL); - if (io->io_pins[pin].io_vector == VECTOR_SMI) + if (io->io_pins[pin].io_irq == IRQ_SMI) return (0); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; - io->io_pins[pin].io_vector = VECTOR_SMI; + io->io_pins[pin].io_irq = IRQ_SMI; io->io_pins[pin].io_masked = 0; io->io_pins[pin].io_edgetrigger = 1; io->io_pins[pin].io_activehi = 1; @@ -674,12 +702,12 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr) return (EINVAL); - if (io->io_pins[pin].io_vector == VECTOR_EXTINT) + if (io->io_pins[pin].io_irq == IRQ_EXTINT) return (0); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; - io->io_pins[pin].io_vector = VECTOR_EXTINT; + io->io_pins[pin].io_irq = IRQ_EXTINT; io->io_pins[pin].io_masked = 1; io->io_pins[pin].io_edgetrigger = 1; io->io_pins[pin].io_activehi = 1; @@ -697,7 +725,7 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr || pol == INTR_POLARITY_CONFORM) return (EINVAL); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_activehi = (pol == INTR_POLARITY_HIGH); if (bootverbose) @@ -714,7 +742,7 @@ io = (struct ioapic *)cookie; if (pin >= io->io_numintr || trigger == INTR_TRIGGER_CONFORM) return (EINVAL); - if (io->io_pins[pin].io_vector >= NUM_IO_INTS) + if (io->io_pins[pin].io_irq >= NUM_IO_INTS) return (EINVAL); io->io_pins[pin].io_edgetrigger = (trigger == INTR_TRIGGER_EDGE); if (bootverbose) @@ -745,18 +773,11 @@ io->io_id, flags >> 4, flags & 0xf, io->io_intbase, io->io_intbase + io->io_numintr - 1); bsp_id = PCPU_GET(apic_id); - for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) { - /* - * Finish initializing the pins by programming the vectors - * and delivery mode. - */ - if (pin->io_vector == VECTOR_DISABLED) - continue; - ioapic_program_intpin(pin); - if (pin->io_vector >= NUM_IO_INTS) - continue; - intr_register_source(&pin->io_intsrc); - } + + /* Register valid pins as interrupt sources. */ + for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) + if (pin->io_irq < NUM_IO_INTS) + intr_register_source(&pin->io_intsrc); } /* --- //depot/projects/smpng/sys/i386/i386/local_apic.c 2005/04/14 18:55:16 +++ //depot/user/jhb/intr/i386/i386/local_apic.c 2005/04/26 21:00:44 @@ -34,10 +34,14 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/local_apic.c,v 1.15 2005/04/14 05:56:17 jhb Exp $"); +#include "opt_ddb.h" + #include #include #include #include +#include +#include #include #include @@ -53,6 +57,11 @@ #include #include +#ifdef DDB +#include +#include +#endif + /* * We can handle up to 60 APICs via our logical cluster IDs, but currently * the physical IDs on Intel processors up to the Pentium 4 are limited to @@ -70,6 +79,9 @@ #define LAPIC_TIMER_STATHZ_DIVIDER 15 #define LAPIC_TIMER_PROFHZ_DIVIDER 3 +/* Magic IRQ value for the timer. */ +#define IRQ_TIMER (NUM_IO_INTS + 1) + /* * Support for local APICs. Local APICs manage interrupts on each * individual processor as opposed to I/O APICs which receive interrupts @@ -123,6 +135,9 @@ IDTVEC(apic_isr7), /* 224 - 255 */ }; +/* Include IDT_SYSCALL to make indexing easier. */ +static u_int8_t ioint_irqs[APIC_NUM_IOINTS + 1]; + static u_int32_t lapic_timer_divisors[] = { APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 @@ -202,6 +217,7 @@ /* Local APIC timer interrupt. */ setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = IRQ_TIMER; /* XXX: error/thermal interrupts */ } @@ -257,19 +273,6 @@ } void -lapic_enable_intr(u_int irq) -{ - u_int vector; - - vector = apic_irq_to_idt(irq); - KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); - KASSERT(ioint_handlers[vector / 32] != NULL, - ("No ISR handler for IRQ %u", irq)); - setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL, - GSEL(GCODE_SEL, SEL_KPL)); -} - -void lapic_setup(void) { struct lapic *la; @@ -686,30 +689,143 @@ lapic->lvt_timer = value; } -/* Translate between IDT vectors and IRQ vectors. */ +/* XXX: better name and place */ +#define BAND_SIZE 16 + +/* Request a free IDT vector to be used by the specified IRQ. */ u_int -apic_irq_to_idt(u_int irq) +apic_alloc_vector(u_int irq) { u_int vector; + int band, count, i; + int best_band, best_count; + + KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); + + /* + * Search for a free vector. We try to spread APIC vectors out + * across the priority bands (upper 4 bits of vector) to avoid + * running into problems with interrupts getting dropped due to + * the per-band FIFOs overflowing. + */ + mtx_lock_spin(&icu_lock); + best_band = -1; + best_count = BAND_SIZE; + for (band = 0; band < APIC_NUM_IOINTS / BAND_SIZE; band ++) { + + /* Determine number of interrupts in this band. */ + count = 0; + for (i = 0; i < BAND_SIZE; i++) + if (ioint_irqs[band * BAND_SIZE + i] != 0) + count++; + + /* Don't try to use a band that is already full. */ + if (count == BAND_SIZE || + (band == (IDT_SYSCALL - APIC_IO_INTS) / BAND_SIZE && + count == BAND_SIZE - 1)) { + printf("APIC: Warning, priority band %u is full!\n", + band + APIC_IO_INTS / BAND_SIZE); + continue; + } + + /* Check if this is the best band we've seen so far. */ + if (count < best_count) { + best_band = band; + best_count = count; + } + } + + /* See if we found a band to use. */ + if (best_band == -1) + panic("APIC vectors are completely full"); + + /* Now find the first free vector in this band. */ + for (i = 0; i < BAND_SIZE; i++) { + vector = best_band * BAND_SIZE + i; + if (ioint_irqs[vector] != 0) + continue; + if (vector + APIC_IO_INTS == IDT_SYSCALL) + continue; + ioint_irqs[vector] = irq; + mtx_unlock_spin(&icu_lock); + return (vector + APIC_IO_INTS); + } + panic("Couldn't find an APIC vector from band %u", best_band); +} + +void +apic_enable_vector(u_int vector) +{ + + KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); + KASSERT(ioint_handlers[vector / 32] != NULL, + ("No ISR handler for vector %u", vector)); + setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); +} +/* Release an APIC vector when it's no longer in use. */ +void +apic_free_vector(u_int vector, u_int irq) +{ + KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && + vector <= APIC_IO_INTS + APIC_NUM_IOINTS, + ("Vector %u does not map to an IRQ line", vector)); KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); - vector = irq + APIC_IO_INTS; - if (vector >= IDT_SYSCALL) - vector++; - return (vector); + KASSERT(ioint_irqs[vector - APIC_IO_INTS] == irq, ("IRQ mismatch")); + mtx_lock_spin(&icu_lock); + ioint_irqs[vector - APIC_IO_INTS] = 0; + mtx_unlock_spin(&icu_lock); } +/* Map an IDT vector (APIC) to an IRQ (interrupt source). */ u_int apic_idt_to_irq(u_int vector) { KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && - vector <= APIC_IO_INTS + NUM_IO_INTS, + vector <= APIC_IO_INTS + APIC_NUM_IOINTS, ("Vector %u does not map to an IRQ line", vector)); - if (vector > IDT_SYSCALL) - vector--; - return (vector - APIC_IO_INTS); + return (ioint_irqs[vector - APIC_IO_INTS]); +} + +#ifdef DDB +/* + * Dump data about APIC IDT vector mappings. + */ +DB_SHOW_COMMAND(apic, db_show_apic) +{ + struct intsrc *isrc; + int quit, i, verbose; + u_int irq; + + quit = 0; + if (strcmp(modif, "vv") == 0) + verbose = 2; + else if (strcmp(modif, "v") == 0) + verbose = 1; + else + verbose = 0; + db_setup_paging(db_simple_pager, &quit, db_lines_per_page); + for (i = 0; i < APIC_NUM_IOINTS + 1 && !quit; i++) { + irq = ioint_irqs[i]; + if (irq != 0) { + db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); + if (irq == IRQ_TIMER) + db_printf("lapic timer\n"); + else if (irq < NUM_IO_INTS) { + isrc = intr_lookup_source(irq); + if (isrc == NULL || verbose == 0) + db_printf("IRQ %u\n", irq); + else + db_dump_ithread(isrc->is_ithread, + verbose == 2); + } else + db_printf("IRQ %u ???\n", irq); + } + } } +#endif /* * APIC probing support code. This includes code to manage enumerators. --- //depot/projects/smpng/sys/i386/include/apicvar.h 2005/04/14 18:55:16 +++ //depot/user/jhb/intr/i386/include/apicvar.h 2005/04/26 19:27:56 @@ -194,7 +194,9 @@ IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(apic_isr6), IDTVEC(apic_isr7), IDTVEC(spuriousint), IDTVEC(timerint); -u_int apic_irq_to_idt(u_int irq); +u_int apic_alloc_vector(u_int irq); +void apic_enable_vector(u_int vector); +void apic_free_vector(u_int vector, u_int irq); u_int apic_idt_to_irq(u_int vector); void apic_register_enumerator(struct apic_enumerator *enumerator); void *ioapic_create(uintptr_t addr, int32_t id, int intbase); @@ -213,7 +215,6 @@ void lapic_create(u_int apic_id, int boot_cpu); void lapic_disable(void); void lapic_dump(const char *str); -void lapic_enable_intr(u_int vector); void lapic_eoi(void); int lapic_id(void); void lapic_init(uintptr_t addr);