lapic.c 15.6 KB raw

#if defined (__x86_64__) || defined (__i386__)
​
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <sys/lapic.h>
#include <sys/cpu.h>
#include <lib/misc.h>
#include <lib/acpi.h>
#include <mm/pmm.h>
​
#define LAPIC_REG_LVT_CMCI    0x2f0
#define LAPIC_REG_LVT_TIMER   0x320
#define LAPIC_REG_LVT_THERMAL 0x330
#define LAPIC_REG_LVT_PMC     0x340
#define LAPIC_REG_LVT_LINT0   0x350
#define LAPIC_REG_LVT_LINT1   0x360
#define LAPIC_REG_LVT_ERROR   0x370
#define LAPIC_REG_SVR         0x0f0
#define LAPIC_REG_TPR         0x080
#define LAPIC_REG_VERSION     0x030
​
static uint32_t pending_lint0 = UINT32_MAX; // no override
static uint32_t pending_lint1 = UINT32_MAX; // no override
​
static uint32_t lapic_madt_nmi_flags_to_lvt(uint16_t flags) {
    uint32_t lvt = 0x10400; // masked + NMI delivery mode
​
    // Polarity: bits 1:0 of flags
    uint8_t polarity = flags & 0x3;
    if (polarity == 0x3) {
        lvt |= (1 << 13); // active low
    }
    // 0b00 (conforms) and 0b01 (active high) leave bit 13 clear
​
    // Trigger mode: bits 3:2 of flags
    uint8_t trigger = (flags >> 2) & 0x3;
    if (trigger == 0x3) {
        lvt |= (1 << 15); // level triggered
    }
    // 0b00 (conforms) and 0b01 (edge) leave bit 15 clear
​
    return lvt;
}
​
void lapic_prep_lint(struct madt *madt, uint32_t acpi_uid, bool x2apic) {
    pending_lint0 = UINT32_MAX; // no override
    pending_lint1 = UINT32_MAX; // no override
​
    // Walk MADT entries looking for NMI entries
    for (uint8_t *madt_ptr = (uint8_t *)madt->madt_entries_begin;
      (uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
      madt_ptr += *(madt_ptr + 1)) {
        if (*(madt_ptr + 1) == 0
         || (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
            break;
        }
        switch (*madt_ptr) {
            case 4: {
                // Local APIC NMI
                if (*(madt_ptr + 1) < sizeof(struct madt_lapic_nmi)) {
                    continue;
                }
​
                struct madt_lapic_nmi *nmi = (void *)madt_ptr;
​
                // Match all processors (0xff) or specific UID.
                if (nmi->acpi_processor_uid != 0xff
                 && (acpi_uid > 0xfe || nmi->acpi_processor_uid != acpi_uid)) {
                    continue;
                }
​
                uint32_t lvt = lapic_madt_nmi_flags_to_lvt(nmi->flags);
                if (nmi->lint == 0) {
                    pending_lint0 = lvt;
                } else if (nmi->lint == 1) {
                    pending_lint1 = lvt;
                }
                continue;
            }
            case 0x0a: {
                // Local x2APIC NMI
                if (!x2apic) {
                    continue;
                }
                if (*(madt_ptr + 1) < sizeof(struct madt_x2apic_nmi)) {
                    continue;
                }
​
                struct madt_x2apic_nmi *nmi = (void *)madt_ptr;
​
                // Match all processors (0xffffffff) or specific UID
                if (nmi->acpi_processor_uid != 0xffffffff && nmi->acpi_processor_uid != acpi_uid) {
                    continue;
                }
​
                uint32_t lvt = lapic_madt_nmi_flags_to_lvt(nmi->flags);
                if (nmi->lint == 0) {
                    pending_lint0 = lvt;
                } else if (nmi->lint == 1) {
                    pending_lint1 = lvt;
                }
                continue;
            }
        }
    }
}
​
static bool lvt_should_mask(uint32_t lvt) {
    switch ((lvt >> 8) & 7) {
        case 0b000: // Fixed
        case 0b001: // Lowest Priority
        case 0b100: // NMI
        case 0b111: // ExtINT
            return true;
        default:    // SMI, INIT, Reserved
            return false;
    }
}
​
void lapic_configure_handoff_state(void) {
    bool is_x2 = !!(rdmsr(0x1b) & (1 << 10));
​
    uint32_t max_lvt;
    if (is_x2) {
        max_lvt = (x2apic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
    } else {
        max_lvt = (lapic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
    }
​
    uint32_t lvt;
​
    if (is_x2) {
        x2apic_write(LAPIC_REG_SVR, 0x1ff);
        x2apic_write(LAPIC_REG_TPR, 0);
        if (max_lvt >= 6) {
            lvt = x2apic_read(LAPIC_REG_LVT_CMCI);
            if (lvt_should_mask(lvt)) {
                x2apic_write(LAPIC_REG_LVT_CMCI, lvt | (1 << 16));
            }
        }
        lvt = x2apic_read(LAPIC_REG_LVT_TIMER);
        if (lvt_should_mask(lvt)) {
            x2apic_write(LAPIC_REG_LVT_TIMER, lvt | (1 << 16));
        }
        if (max_lvt >= 5) {
            lvt = x2apic_read(LAPIC_REG_LVT_THERMAL);
            if (lvt_should_mask(lvt)) {
                x2apic_write(LAPIC_REG_LVT_THERMAL, lvt | (1 << 16));
            }
        }
        if (max_lvt >= 4) {
            lvt = x2apic_read(LAPIC_REG_LVT_PMC);
            if (lvt_should_mask(lvt)) {
                x2apic_write(LAPIC_REG_LVT_PMC, lvt | (1 << 16));
            }
        }
        lvt = x2apic_read(LAPIC_REG_LVT_ERROR);
        if (lvt_should_mask(lvt)) {
            x2apic_write(LAPIC_REG_LVT_ERROR, lvt | (1 << 16));
        }
        lvt = x2apic_read(LAPIC_REG_LVT_LINT0);
        if (lvt_should_mask(lvt)) {
            x2apic_write(LAPIC_REG_LVT_LINT0, pending_lint0 != UINT32_MAX ? pending_lint0 : lvt | (1 << 16));
        }
        lvt = x2apic_read(LAPIC_REG_LVT_LINT1);
        if (lvt_should_mask(lvt)) {
            x2apic_write(LAPIC_REG_LVT_LINT1, pending_lint1 != UINT32_MAX ? pending_lint1 : lvt | (1 << 16));
        }
    } else {
        lapic_write(LAPIC_REG_SVR, 0x1ff);
        lapic_write(LAPIC_REG_TPR, 0);
        if (max_lvt >= 6) {
            lvt = lapic_read(LAPIC_REG_LVT_CMCI);
            if (lvt_should_mask(lvt)) {
                lapic_write(LAPIC_REG_LVT_CMCI, lvt | (1 << 16));
            }
        }
        lvt = lapic_read(LAPIC_REG_LVT_TIMER);
        if (lvt_should_mask(lvt)) {
            lapic_write(LAPIC_REG_LVT_TIMER, lvt | (1 << 16));
        }
        if (max_lvt >= 5) {
            lvt = lapic_read(LAPIC_REG_LVT_THERMAL);
            if (lvt_should_mask(lvt)) {
                lapic_write(LAPIC_REG_LVT_THERMAL, lvt | (1 << 16));
            }
        }
        if (max_lvt >= 4) {
            lvt = lapic_read(LAPIC_REG_LVT_PMC);
            if (lvt_should_mask(lvt)) {
                lapic_write(LAPIC_REG_LVT_PMC, lvt | (1 << 16));
            }
        }
        lvt = lapic_read(LAPIC_REG_LVT_ERROR);
        if (lvt_should_mask(lvt)) {
            lapic_write(LAPIC_REG_LVT_ERROR, lvt | (1 << 16));
        }
        lvt = lapic_read(LAPIC_REG_LVT_LINT0);
        if (lvt_should_mask(lvt)) {
            lapic_write(LAPIC_REG_LVT_LINT0, pending_lint0 != UINT32_MAX ? pending_lint0 : lvt | (1 << 16));
        }
        lvt = lapic_read(LAPIC_REG_LVT_LINT1);
        if (lvt_should_mask(lvt)) {
            lapic_write(LAPIC_REG_LVT_LINT1, pending_lint1 != UINT32_MAX ? pending_lint1 : lvt | (1 << 16));
        }
    }
}
​
void lapic_configure_bsp(void) {
    struct madt *madt = acpi_get_table("APIC", 0);
    if (madt == NULL) {
        return;
    }
​
    // Detect x2APIC from MSR
    bool is_x2 = !!(rdmsr(0x1b) & (1 << 10));
​
    // Find the BSP entry by matching LAPIC ID
    uint32_t bsp_lapic_id;
    if (is_x2) {
        bsp_lapic_id = x2apic_read(LAPIC_REG_ID);
    } else {
        bsp_lapic_id = lapic_read(LAPIC_REG_ID) >> 24;
    }
​
    uint32_t bsp_acpi_uid = 0;
​
    for (uint8_t *madt_ptr = (uint8_t *)madt->madt_entries_begin;
      (uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
      madt_ptr += *(madt_ptr + 1)) {
        if (*(madt_ptr + 1) == 0
         || (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
            break;
        }
        switch (*madt_ptr) {
            case 0: {
                if (*(madt_ptr + 1) < sizeof(struct madt_lapic)) {
                    continue;
                }
                struct madt_lapic *lapic = (void *)madt_ptr;
                if (lapic->lapic_id == bsp_lapic_id) {
                    bsp_acpi_uid = lapic->acpi_processor_uid;
                    goto found;
                }
                continue;
            }
            case 9: {
                if (!is_x2) {
                    continue;
                }
                if (*(madt_ptr + 1) < sizeof(struct madt_x2apic)) {
                    continue;
                }
                struct madt_x2apic *x2lapic = (void *)madt_ptr;
                if (x2lapic->x2apic_id == bsp_lapic_id) {
                    bsp_acpi_uid = x2lapic->acpi_processor_uid;
                    goto found;
                }
                continue;
            }
        }
    }
​
found:
    lapic_prep_lint(madt, bsp_acpi_uid, is_x2);
    lapic_configure_handoff_state();
}
​
struct dmar {
    struct sdt header;
    uint8_t host_address_width;
    uint8_t flags;
    uint8_t reserved[10];
    symbol  remapping_structures;
} __attribute__((packed));
​
bool lapic_check(void) {
    uint32_t eax, ebx, ecx, edx;
    if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx))
        return false;
​
    if (!(edx & (1 << 9)))
        return false;
​
    return true;
}
​
uint32_t lapic_read(uint32_t reg) {
    size_t lapic_mmio_base = (size_t)(rdmsr(0x1b) & 0xfffffffffffff000);
    return mmind(lapic_mmio_base + reg);
}
​
void lapic_write(uint32_t reg, uint32_t data) {
    size_t lapic_mmio_base = (size_t)(rdmsr(0x1b) & 0xfffffffffffff000);
    mmoutd(lapic_mmio_base + reg, data);
}
​
void lapic_icr_wait(void) {
    for (int i = 0; i < 1000000; i++) {
        if (!(lapic_read(LAPIC_REG_ICR0) & (1 << 12))) {
            return;
        }
        asm volatile ("pause");
    }
}
​
bool x2apic_check(void) {
    uint32_t eax, ebx, ecx, edx;
    if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx))
        return false;
​
    if (!(ecx & (1 << 21)))
        return false;
​
    // According to the Intel VT-d spec, we're required
    // to check if bit 0 and 1 of the flags field of the
    // DMAR ACPI table are set, and if they are, we should
    // not report x2APIC capabilities.
    struct dmar *dmar = acpi_get_table("DMAR", 0);
    if (!dmar)
        return true;
​
    if ((dmar->flags & (1 << 0)) && (dmar->flags & (1 << 1)))
        return false;
​
    return true;
}
​
static bool x2apic_mode = false;
​
bool x2apic_enable(void) {
    if (!x2apic_check())
        return false;
​
    uint64_t ia32_apic_base = rdmsr(0x1b);
    ia32_apic_base |= (1 << 10);
    wrmsr(0x1b, ia32_apic_base);
​
    x2apic_mode = true;
​
    return true;
}
​
bool x2apic_disable(void) {
    uint64_t msr = rdmsr(0x1b);
    if (!(msr & (1 << 10)))
        return true;
​
    // Check for LEGACY_XAPIC_DISABLED (Intel Meteor Lake+).
    // CPUID.07H.0:EDX[29] enumerates IA32_ARCH_CAPABILITIES MSR (0x10A).
    // IA32_ARCH_CAPABILITIES bit 21 = XAPIC_DISABLE feature supported.
    // IA32_XAPIC_DISABLE_STATUS MSR (0xBD) bit 0 = xAPIC permanently disabled.
    uint32_t eax, ebx, ecx, edx;
    if (cpuid(7, 0, &eax, &ebx, &ecx, &edx) && (edx & (1 << 29))) {
        uint64_t arch_caps = rdmsr(0x10a);
        if (arch_caps & (1 << 21)) {
            if (rdmsr(0xbd) & 1) {
                return false;
            }
        }
    }
​
    // Save LAPIC state; clearing EN resets all registers except APIC ID.
    uint32_t max_lvt = (x2apic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
    uint32_t saved_svr = x2apic_read(LAPIC_REG_SVR);
    uint32_t saved_tpr = x2apic_read(LAPIC_REG_TPR);
    uint32_t saved_timer = x2apic_read(LAPIC_REG_LVT_TIMER);
    uint32_t saved_lint0 = x2apic_read(LAPIC_REG_LVT_LINT0);
    uint32_t saved_lint1 = x2apic_read(LAPIC_REG_LVT_LINT1);
    uint32_t saved_error = x2apic_read(LAPIC_REG_LVT_ERROR);
    uint32_t saved_pmc = max_lvt >= 4 ? x2apic_read(LAPIC_REG_LVT_PMC) : 0;
    uint32_t saved_thermal = max_lvt >= 5 ? x2apic_read(LAPIC_REG_LVT_THERMAL) : 0;
    uint32_t saved_cmci = max_lvt >= 6 ? x2apic_read(LAPIC_REG_LVT_CMCI) : 0;
​
    // Transition x2APIC -> disabled -> xAPIC.
    // Direct x2APIC -> xAPIC is an invalid transition (#GP).
    msr &= ~((1ULL << 11) | (1ULL << 10));
    wrmsr(0x1b, msr);
​
    msr |= (1ULL << 11);
    wrmsr(0x1b, msr);
​
    x2apic_mode = false;
​
    // Restore LAPIC state. SVR is restored last to re-enable the APIC.
    lapic_write(LAPIC_REG_TPR, saved_tpr);
    lapic_write(LAPIC_REG_LVT_TIMER, saved_timer);
    lapic_write(LAPIC_REG_LVT_LINT0, saved_lint0);
    lapic_write(LAPIC_REG_LVT_LINT1, saved_lint1);
    lapic_write(LAPIC_REG_LVT_ERROR, saved_error);
    if (max_lvt >= 4) lapic_write(LAPIC_REG_LVT_PMC, saved_pmc);
    if (max_lvt >= 5) lapic_write(LAPIC_REG_LVT_THERMAL, saved_thermal);
    if (max_lvt >= 6) lapic_write(LAPIC_REG_LVT_CMCI, saved_cmci);
    lapic_write(LAPIC_REG_SVR, saved_svr);
​
    return true;
}
​
void lapic_eoi(void) {
    if (!x2apic_mode) {
        lapic_write(0xb0, 0);
    } else {
        x2apic_write(0xb0, 0);
    }
}
​
uint64_t x2apic_read(uint32_t reg) {
    return rdmsr(0x800 + (reg >> 4));
}
​
void x2apic_write(uint32_t reg, uint64_t data) {
    wrmsr(0x800 + (reg >> 4), data);
}
​
static struct madt_io_apic **io_apics = NULL;
static size_t max_io_apics = 0;
​
void init_io_apics(void) {
    static bool already_inited = false;
    if (already_inited) {
        return;
    }
​
    struct madt *madt = acpi_get_table("APIC", 0);
​
    if (madt == NULL) {
        goto out;
    }
​
    for (uint8_t *madt_ptr = (uint8_t *)madt->madt_entries_begin;
      (uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
      madt_ptr += *(madt_ptr + 1)) {
        if (*(madt_ptr + 1) == 0
         || (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
            break;
        }
        switch (*madt_ptr) {
            case 1: {
                if (*(madt_ptr + 1) < sizeof(struct madt_io_apic))
                    continue;
                max_io_apics++;
                continue;
            }
        }
    }
​
    io_apics = ext_mem_alloc_counted(max_io_apics, sizeof(struct madt_io_apic *));
    max_io_apics = 0;
​
    for (uint8_t *madt_ptr = (uint8_t *)madt->madt_entries_begin;
      (uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
      madt_ptr += *(madt_ptr + 1)) {
        if (*(madt_ptr + 1) == 0
         || (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
            break;
        }
        switch (*madt_ptr) {
            case 1: {
                if (*(madt_ptr + 1) < sizeof(struct madt_io_apic))
                    continue;
                io_apics[max_io_apics++] = (void *)madt_ptr;
                continue;
            }
        }
    }
​
out:
    already_inited = true;
}
​
uint32_t io_apic_read(size_t io_apic, uint32_t reg) {
    uintptr_t base = (uintptr_t)io_apics[io_apic]->address;
    mmoutd(base, reg);
    return mmind(base + 16);
}
​
void io_apic_write(size_t io_apic, uint32_t reg, uint32_t value) {
    uintptr_t base = (uintptr_t)io_apics[io_apic]->address;
    mmoutd(base, reg);
    mmoutd(base + 16, value);
}
​
uint32_t io_apic_gsi_count(size_t io_apic) {
    return ((io_apic_read(io_apic, 1) & 0xff0000) >> 16) + 1;
}
​
void io_apic_mask_all(bool mask_nmi_and_extint) {
    for (size_t i = 0; i < max_io_apics; i++) {
        uint32_t gsi_count = io_apic_gsi_count(i);
        for (uint32_t j = 0; j < gsi_count; j++) {
            uintptr_t ioredtbl = j * 2 + 16;
            switch ((io_apic_read(i, ioredtbl) >> 8) & 0b111) {
                case 0b000: // Fixed
                case 0b001: // Lowest Priority
                    break;
                case 0b100: // NMI
                case 0b111: // ExtINT
                    if (!mask_nmi_and_extint) {
                        continue;
                    }
                    break;
                default:
                    continue;
            }
​
            io_apic_write(i, ioredtbl, io_apic_read(i, ioredtbl) | (1 << 16));
        }
    }
}
​
#endif

tab: 2 4 8 wrap: off on

1	#if defined (__x86_64__) \|\| defined (__i386__)
2
3	#include <stdint.h>
4	#include <stddef.h>
5	#include <stdbool.h>
6	#include <sys/lapic.h>
7	#include <sys/cpu.h>
8	#include <lib/misc.h>
9	#include <lib/acpi.h>
10	#include <mm/pmm.h>
11
12	#define LAPIC_REG_LVT_CMCI 0x2f0
13	#define LAPIC_REG_LVT_TIMER 0x320
14	#define LAPIC_REG_LVT_THERMAL 0x330
15	#define LAPIC_REG_LVT_PMC 0x340
16	#define LAPIC_REG_LVT_LINT0 0x350
17	#define LAPIC_REG_LVT_LINT1 0x360
18	#define LAPIC_REG_LVT_ERROR 0x370
19	#define LAPIC_REG_SVR 0x0f0
20	#define LAPIC_REG_TPR 0x080
21	#define LAPIC_REG_VERSION 0x030
22
23	static uint32_t pending_lint0 = UINT32_MAX; // no override
24	static uint32_t pending_lint1 = UINT32_MAX; // no override
25
26	static uint32_t lapic_madt_nmi_flags_to_lvt(uint16_t flags) {
27	uint32_t lvt = 0x10400; // masked + NMI delivery mode
28
29	// Polarity: bits 1:0 of flags
30	uint8_t polarity = flags & 0x3;
31	if (polarity == 0x3) {
32	lvt \|= (1 << 13); // active low
33	}
34	// 0b00 (conforms) and 0b01 (active high) leave bit 13 clear
35
36	// Trigger mode: bits 3:2 of flags
37	uint8_t trigger = (flags >> 2) & 0x3;
38	if (trigger == 0x3) {
39	lvt \|= (1 << 15); // level triggered
40	}
41	// 0b00 (conforms) and 0b01 (edge) leave bit 15 clear
42
43	return lvt;
44	}
45
46	void lapic_prep_lint(struct madt *madt, uint32_t acpi_uid, bool x2apic) {
47	pending_lint0 = UINT32_MAX; // no override
48	pending_lint1 = UINT32_MAX; // no override
49
50	// Walk MADT entries looking for NMI entries
51	for (uint8_t madt_ptr = (uint8_t )madt->madt_entries_begin;
52	(uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
53	madt_ptr += *(madt_ptr + 1)) {
54	if (*(madt_ptr + 1) == 0
55	\|\| (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
56	break;
57	}
58	switch (*madt_ptr) {
59	case 4: {
60	// Local APIC NMI
61	if (*(madt_ptr + 1) < sizeof(struct madt_lapic_nmi)) {
62	continue;
63	}
64
65	struct madt_lapic_nmi nmi = (void )madt_ptr;
66
67	// Match all processors (0xff) or specific UID.
68	if (nmi->acpi_processor_uid != 0xff
69	&& (acpi_uid > 0xfe \|\| nmi->acpi_processor_uid != acpi_uid)) {
70	continue;
71	}
72
73	uint32_t lvt = lapic_madt_nmi_flags_to_lvt(nmi->flags);
74	if (nmi->lint == 0) {
75	pending_lint0 = lvt;
76	} else if (nmi->lint == 1) {
77	pending_lint1 = lvt;
78	}
79	continue;
80	}
81	case 0x0a: {
82	// Local x2APIC NMI
83	if (!x2apic) {
84	continue;
85	}
86	if (*(madt_ptr + 1) < sizeof(struct madt_x2apic_nmi)) {
87	continue;
88	}
89
90	struct madt_x2apic_nmi nmi = (void )madt_ptr;
91
92	// Match all processors (0xffffffff) or specific UID
93	if (nmi->acpi_processor_uid != 0xffffffff && nmi->acpi_processor_uid != acpi_uid) {
94	continue;
95	}
96
97	uint32_t lvt = lapic_madt_nmi_flags_to_lvt(nmi->flags);
98	if (nmi->lint == 0) {
99	pending_lint0 = lvt;
100	} else if (nmi->lint == 1) {
101	pending_lint1 = lvt;
102	}
103	continue;
104	}
105	}
106	}
107	}
108
109	static bool lvt_should_mask(uint32_t lvt) {
110	switch ((lvt >> 8) & 7) {
111	case 0b000: // Fixed
112	case 0b001: // Lowest Priority
113	case 0b100: // NMI
114	case 0b111: // ExtINT
115	return true;
116	default: // SMI, INIT, Reserved
117	return false;
118	}
119	}
120
121	void lapic_configure_handoff_state(void) {
122	bool is_x2 = !!(rdmsr(0x1b) & (1 << 10));
123
124	uint32_t max_lvt;
125	if (is_x2) {
126	max_lvt = (x2apic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
127	} else {
128	max_lvt = (lapic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
129	}
130
131	uint32_t lvt;
132
133	if (is_x2) {
134	x2apic_write(LAPIC_REG_SVR, 0x1ff);
135	x2apic_write(LAPIC_REG_TPR, 0);
136	if (max_lvt >= 6) {
137	lvt = x2apic_read(LAPIC_REG_LVT_CMCI);
138	if (lvt_should_mask(lvt)) {
139	x2apic_write(LAPIC_REG_LVT_CMCI, lvt \| (1 << 16));
140	}
141	}
142	lvt = x2apic_read(LAPIC_REG_LVT_TIMER);
143	if (lvt_should_mask(lvt)) {
144	x2apic_write(LAPIC_REG_LVT_TIMER, lvt \| (1 << 16));
145	}
146	if (max_lvt >= 5) {
147	lvt = x2apic_read(LAPIC_REG_LVT_THERMAL);
148	if (lvt_should_mask(lvt)) {
149	x2apic_write(LAPIC_REG_LVT_THERMAL, lvt \| (1 << 16));
150	}
151	}
152	if (max_lvt >= 4) {
153	lvt = x2apic_read(LAPIC_REG_LVT_PMC);
154	if (lvt_should_mask(lvt)) {
155	x2apic_write(LAPIC_REG_LVT_PMC, lvt \| (1 << 16));
156	}
157	}
158	lvt = x2apic_read(LAPIC_REG_LVT_ERROR);
159	if (lvt_should_mask(lvt)) {
160	x2apic_write(LAPIC_REG_LVT_ERROR, lvt \| (1 << 16));
161	}
162	lvt = x2apic_read(LAPIC_REG_LVT_LINT0);
163	if (lvt_should_mask(lvt)) {
164	x2apic_write(LAPIC_REG_LVT_LINT0, pending_lint0 != UINT32_MAX ? pending_lint0 : lvt \| (1 << 16));
165	}
166	lvt = x2apic_read(LAPIC_REG_LVT_LINT1);
167	if (lvt_should_mask(lvt)) {
168	x2apic_write(LAPIC_REG_LVT_LINT1, pending_lint1 != UINT32_MAX ? pending_lint1 : lvt \| (1 << 16));
169	}
170	} else {
171	lapic_write(LAPIC_REG_SVR, 0x1ff);
172	lapic_write(LAPIC_REG_TPR, 0);
173	if (max_lvt >= 6) {
174	lvt = lapic_read(LAPIC_REG_LVT_CMCI);
175	if (lvt_should_mask(lvt)) {
176	lapic_write(LAPIC_REG_LVT_CMCI, lvt \| (1 << 16));
177	}
178	}
179	lvt = lapic_read(LAPIC_REG_LVT_TIMER);
180	if (lvt_should_mask(lvt)) {
181	lapic_write(LAPIC_REG_LVT_TIMER, lvt \| (1 << 16));
182	}
183	if (max_lvt >= 5) {
184	lvt = lapic_read(LAPIC_REG_LVT_THERMAL);
185	if (lvt_should_mask(lvt)) {
186	lapic_write(LAPIC_REG_LVT_THERMAL, lvt \| (1 << 16));
187	}
188	}
189	if (max_lvt >= 4) {
190	lvt = lapic_read(LAPIC_REG_LVT_PMC);
191	if (lvt_should_mask(lvt)) {
192	lapic_write(LAPIC_REG_LVT_PMC, lvt \| (1 << 16));
193	}
194	}
195	lvt = lapic_read(LAPIC_REG_LVT_ERROR);
196	if (lvt_should_mask(lvt)) {
197	lapic_write(LAPIC_REG_LVT_ERROR, lvt \| (1 << 16));
198	}
199	lvt = lapic_read(LAPIC_REG_LVT_LINT0);
200	if (lvt_should_mask(lvt)) {
201	lapic_write(LAPIC_REG_LVT_LINT0, pending_lint0 != UINT32_MAX ? pending_lint0 : lvt \| (1 << 16));
202	}
203	lvt = lapic_read(LAPIC_REG_LVT_LINT1);
204	if (lvt_should_mask(lvt)) {
205	lapic_write(LAPIC_REG_LVT_LINT1, pending_lint1 != UINT32_MAX ? pending_lint1 : lvt \| (1 << 16));
206	}
207	}
208	}
209
210	void lapic_configure_bsp(void) {
211	struct madt *madt = acpi_get_table("APIC", 0);
212	if (madt == NULL) {
213	return;
214	}
215
216	// Detect x2APIC from MSR
217	bool is_x2 = !!(rdmsr(0x1b) & (1 << 10));
218
219	// Find the BSP entry by matching LAPIC ID
220	uint32_t bsp_lapic_id;
221	if (is_x2) {
222	bsp_lapic_id = x2apic_read(LAPIC_REG_ID);
223	} else {
224	bsp_lapic_id = lapic_read(LAPIC_REG_ID) >> 24;
225	}
226
227	uint32_t bsp_acpi_uid = 0;
228
229	for (uint8_t madt_ptr = (uint8_t )madt->madt_entries_begin;
230	(uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
231	madt_ptr += *(madt_ptr + 1)) {
232	if (*(madt_ptr + 1) == 0
233	\|\| (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
234	break;
235	}
236	switch (*madt_ptr) {
237	case 0: {
238	if (*(madt_ptr + 1) < sizeof(struct madt_lapic)) {
239	continue;
240	}
241	struct madt_lapic lapic = (void )madt_ptr;
242	if (lapic->lapic_id == bsp_lapic_id) {
243	bsp_acpi_uid = lapic->acpi_processor_uid;
244	goto found;
245	}
246	continue;
247	}
248	case 9: {
249	if (!is_x2) {
250	continue;
251	}
252	if (*(madt_ptr + 1) < sizeof(struct madt_x2apic)) {
253	continue;
254	}
255	struct madt_x2apic x2lapic = (void )madt_ptr;
256	if (x2lapic->x2apic_id == bsp_lapic_id) {
257	bsp_acpi_uid = x2lapic->acpi_processor_uid;
258	goto found;
259	}
260	continue;
261	}
262	}
263	}
264
265	found:
266	lapic_prep_lint(madt, bsp_acpi_uid, is_x2);
267	lapic_configure_handoff_state();
268	}
269
270	struct dmar {
271	struct sdt header;
272	uint8_t host_address_width;
273	uint8_t flags;
274	uint8_t reserved[10];
275	symbol remapping_structures;
276	} __attribute__((packed));
277
278	bool lapic_check(void) {
279	uint32_t eax, ebx, ecx, edx;
280	if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx))
281	return false;
282
283	if (!(edx & (1 << 9)))
284	return false;
285
286	return true;
287	}
288
289	uint32_t lapic_read(uint32_t reg) {
290	size_t lapic_mmio_base = (size_t)(rdmsr(0x1b) & 0xfffffffffffff000);
291	return mmind(lapic_mmio_base + reg);
292	}
293
294	void lapic_write(uint32_t reg, uint32_t data) {
295	size_t lapic_mmio_base = (size_t)(rdmsr(0x1b) & 0xfffffffffffff000);
296	mmoutd(lapic_mmio_base + reg, data);
297	}
298
299	void lapic_icr_wait(void) {
300	for (int i = 0; i < 1000000; i++) {
301	if (!(lapic_read(LAPIC_REG_ICR0) & (1 << 12))) {
302	return;
303	}
304	asm volatile ("pause");
305	}
306	}
307
308	bool x2apic_check(void) {
309	uint32_t eax, ebx, ecx, edx;
310	if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx))
311	return false;
312
313	if (!(ecx & (1 << 21)))
314	return false;
315
316	// According to the Intel VT-d spec, we're required
317	// to check if bit 0 and 1 of the flags field of the
318	// DMAR ACPI table are set, and if they are, we should
319	// not report x2APIC capabilities.
320	struct dmar *dmar = acpi_get_table("DMAR", 0);
321	if (!dmar)
322	return true;
323
324	if ((dmar->flags & (1 << 0)) && (dmar->flags & (1 << 1)))
325	return false;
326
327	return true;
328	}
329
330	static bool x2apic_mode = false;
331
332	bool x2apic_enable(void) {
333	if (!x2apic_check())
334	return false;
335
336	uint64_t ia32_apic_base = rdmsr(0x1b);
337	ia32_apic_base \|= (1 << 10);
338	wrmsr(0x1b, ia32_apic_base);
339
340	x2apic_mode = true;
341
342	return true;
343	}
344
345	bool x2apic_disable(void) {
346	uint64_t msr = rdmsr(0x1b);
347	if (!(msr & (1 << 10)))
348	return true;
349
350	// Check for LEGACY_XAPIC_DISABLED (Intel Meteor Lake+).
351	// CPUID.07H.0:EDX[29] enumerates IA32_ARCH_CAPABILITIES MSR (0x10A).
352	// IA32_ARCH_CAPABILITIES bit 21 = XAPIC_DISABLE feature supported.
353	// IA32_XAPIC_DISABLE_STATUS MSR (0xBD) bit 0 = xAPIC permanently disabled.
354	uint32_t eax, ebx, ecx, edx;
355	if (cpuid(7, 0, &eax, &ebx, &ecx, &edx) && (edx & (1 << 29))) {
356	uint64_t arch_caps = rdmsr(0x10a);
357	if (arch_caps & (1 << 21)) {
358	if (rdmsr(0xbd) & 1) {
359	return false;
360	}
361	}
362	}
363
364	// Save LAPIC state; clearing EN resets all registers except APIC ID.
365	uint32_t max_lvt = (x2apic_read(LAPIC_REG_VERSION) >> 16) & 0xff;
366	uint32_t saved_svr = x2apic_read(LAPIC_REG_SVR);
367	uint32_t saved_tpr = x2apic_read(LAPIC_REG_TPR);
368	uint32_t saved_timer = x2apic_read(LAPIC_REG_LVT_TIMER);
369	uint32_t saved_lint0 = x2apic_read(LAPIC_REG_LVT_LINT0);
370	uint32_t saved_lint1 = x2apic_read(LAPIC_REG_LVT_LINT1);
371	uint32_t saved_error = x2apic_read(LAPIC_REG_LVT_ERROR);
372	uint32_t saved_pmc = max_lvt >= 4 ? x2apic_read(LAPIC_REG_LVT_PMC) : 0;
373	uint32_t saved_thermal = max_lvt >= 5 ? x2apic_read(LAPIC_REG_LVT_THERMAL) : 0;
374	uint32_t saved_cmci = max_lvt >= 6 ? x2apic_read(LAPIC_REG_LVT_CMCI) : 0;
375
376	// Transition x2APIC -> disabled -> xAPIC.
377	// Direct x2APIC -> xAPIC is an invalid transition (#GP).
378	msr &= ~((1ULL << 11) \| (1ULL << 10));
379	wrmsr(0x1b, msr);
380
381	msr \|= (1ULL << 11);
382	wrmsr(0x1b, msr);
383
384	x2apic_mode = false;
385
386	// Restore LAPIC state. SVR is restored last to re-enable the APIC.
387	lapic_write(LAPIC_REG_TPR, saved_tpr);
388	lapic_write(LAPIC_REG_LVT_TIMER, saved_timer);
389	lapic_write(LAPIC_REG_LVT_LINT0, saved_lint0);
390	lapic_write(LAPIC_REG_LVT_LINT1, saved_lint1);
391	lapic_write(LAPIC_REG_LVT_ERROR, saved_error);
392	if (max_lvt >= 4) lapic_write(LAPIC_REG_LVT_PMC, saved_pmc);
393	if (max_lvt >= 5) lapic_write(LAPIC_REG_LVT_THERMAL, saved_thermal);
394	if (max_lvt >= 6) lapic_write(LAPIC_REG_LVT_CMCI, saved_cmci);
395	lapic_write(LAPIC_REG_SVR, saved_svr);
396
397	return true;
398	}
399
400	void lapic_eoi(void) {
401	if (!x2apic_mode) {
402	lapic_write(0xb0, 0);
403	} else {
404	x2apic_write(0xb0, 0);
405	}
406	}
407
408	uint64_t x2apic_read(uint32_t reg) {
409	return rdmsr(0x800 + (reg >> 4));
410	}
411
412	void x2apic_write(uint32_t reg, uint64_t data) {
413	wrmsr(0x800 + (reg >> 4), data);
414	}
415
416	static struct madt_io_apic **io_apics = NULL;
417	static size_t max_io_apics = 0;
418
419	void init_io_apics(void) {
420	static bool already_inited = false;
421	if (already_inited) {
422	return;
423	}
424
425	struct madt *madt = acpi_get_table("APIC", 0);
426
427	if (madt == NULL) {
428	goto out;
429	}
430
431	for (uint8_t madt_ptr = (uint8_t )madt->madt_entries_begin;
432	(uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
433	madt_ptr += *(madt_ptr + 1)) {
434	if (*(madt_ptr + 1) == 0
435	\|\| (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
436	break;
437	}
438	switch (*madt_ptr) {
439	case 1: {
440	if (*(madt_ptr + 1) < sizeof(struct madt_io_apic))
441	continue;
442	max_io_apics++;
443	continue;
444	}
445	}
446	}
447
448	io_apics = ext_mem_alloc_counted(max_io_apics, sizeof(struct madt_io_apic *));
449	max_io_apics = 0;
450
451	for (uint8_t madt_ptr = (uint8_t )madt->madt_entries_begin;
452	(uintptr_t)madt_ptr + 1 < (uintptr_t)madt + madt->header.length;
453	madt_ptr += *(madt_ptr + 1)) {
454	if (*(madt_ptr + 1) == 0
455	\|\| (uintptr_t)madt_ptr + *(madt_ptr + 1) > (uintptr_t)madt + madt->header.length) {
456	break;
457	}
458	switch (*madt_ptr) {
459	case 1: {
460	if (*(madt_ptr + 1) < sizeof(struct madt_io_apic))
461	continue;
462	io_apics[max_io_apics++] = (void *)madt_ptr;
463	continue;
464	}
465	}
466	}
467
468	out:
469	already_inited = true;
470	}
471
472	uint32_t io_apic_read(size_t io_apic, uint32_t reg) {
473	uintptr_t base = (uintptr_t)io_apics[io_apic]->address;
474	mmoutd(base, reg);
475	return mmind(base + 16);
476	}
477
478	void io_apic_write(size_t io_apic, uint32_t reg, uint32_t value) {
479	uintptr_t base = (uintptr_t)io_apics[io_apic]->address;
480	mmoutd(base, reg);
481	mmoutd(base + 16, value);
482	}
483
484	uint32_t io_apic_gsi_count(size_t io_apic) {
485	return ((io_apic_read(io_apic, 1) & 0xff0000) >> 16) + 1;
486	}
487
488	void io_apic_mask_all(bool mask_nmi_and_extint) {
489	for (size_t i = 0; i < max_io_apics; i++) {
490	uint32_t gsi_count = io_apic_gsi_count(i);
491	for (uint32_t j = 0; j < gsi_count; j++) {
492	uintptr_t ioredtbl = j * 2 + 16;
493	switch ((io_apic_read(i, ioredtbl) >> 8) & 0b111) {
494	case 0b000: // Fixed
495	case 0b001: // Lowest Priority
496	break;
497	case 0b100: // NMI
498	case 0b111: // ExtINT
499	if (!mask_nmi_and_extint) {
500	continue;
501	}
502	break;
503	default:
504	continue;
505	}
506
507	io_apic_write(i, ioredtbl, io_apic_read(i, ioredtbl) \| (1 << 16));
508	}
509	}
510	}
511
512	#endif