:: commit 2c8c0c3deb34f544c227b5148396435293987fc2
Revert "misc: Converge with 5.x"
This reverts commit f9682543fd6ba603ced0135053893c66712a857b.
diff --git a/common/lib/misc.h b/common/lib/misc.h
index 9f1d372b..495c7984 100644
--- a/common/lib/misc.h
+++ b/common/lib/misc.h
@@ -99,7 +99,7 @@ noreturn void enter_in_el1(uint64_t entry, uint64_t sp, uint64_t sctlr,
uint64_t mair, uint64_t tcr, uint64_t ttbr0,
uint64_t ttbr1, uint64_t target_x0);
#elif defined (__riscv64)
-noreturn void riscv_spinup(uint64_t entry, uint64_t sp, uint64_t satp);
+noreturn void riscv_spinup(uint64_t entry, uint64_t sp, uint64_t satp, uint64_t direct_map_offset);
#if defined (UEFI)
RISCV_EFI_BOOT_PROTOCOL *get_riscv_boot_protocol(void);
#endif
diff --git a/common/lib/spinup.asm_aarch64 b/common/lib/spinup.asm_aarch64
index 8d8a2e7f..20fbf235 100644
--- a/common/lib/spinup.asm_aarch64
+++ b/common/lib/spinup.asm_aarch64
@@ -4,7 +4,7 @@
// noreturn void enter_in_el1(uint64_t entry, uint64_t sp, uint64_t sctlr,
// uint64_t mair, uint64_t tcr, uint64_t ttbr0,
-// uint64_t ttbr1, uint64_t target_x0)
+// uint64_t ttbr1, uint64_t direct_map_offset)
// Potentially drop to EL1 from EL2 (and also disable trapping to EL2), then
// configure EL1 state and jump to kernel.
@@ -13,28 +13,59 @@ enter_in_el1:
msr spsel, #0
mov sp, x1
- // Configure EL1 state
+ PICK_EL x8, 0f, 2f
+0:
+ // Switch to the new page tables
+
+ // Point the EL1t handler to the continuation, such that after we page fault,
+ // execution continues and the kernel is entered.
+ adrp x8, 1f
+ add x8, x8, #:lo12:1f
+ add x8, x8, x7
+ msr vbar_el1, x8
+ isb
+ dsb sy
+ isb
+
+ // Switch the page table registers
msr mair_el1, x3
msr tcr_el1, x4
msr ttbr0_el1, x5
msr ttbr1_el1, x6
msr sctlr_el1, x2
+ isb
dsb sy
isb
- PICK_EL x8, 0f, 1f
-0:
+ // Jump to the higher half mapping in case we didn't immediately crash
+ br x8
+
+// Alignment required by VBAR register
+.align 11
+1:
+ // Zero out VBAR to avoid confusion
+ msr vbar_el1, xzr
+
// Enter kernel in EL1
mov x8, #0x3c4
msr spsr_el1, x8
msr elr_el1, x0
- mov x0, x7
+ mov x0, xzr
ZERO_REGS_EXCEPT_X0
eret
-1:
+2:
+ // Configure EL1 state
+ msr mair_el1, x3
+ msr tcr_el1, x4
+ msr ttbr0_el1, x5
+ msr ttbr1_el1, x6
+ msr sctlr_el1, x2
+ dsb sy
+ isb
+
// Configure EL2-specific state for EL1
// Don't trap counters to EL2
@@ -57,7 +88,7 @@ enter_in_el1:
msr spsr_el2, x8
msr elr_el2, x0
- mov x0, x7
+ mov x0, xzr
ZERO_REGS_EXCEPT_X0
eret
diff --git a/common/lib/spinup.asm_riscv64 b/common/lib/spinup.asm_riscv64
index c5eaebd7..8e964ee3 100644
--- a/common/lib/spinup.asm_riscv64
+++ b/common/lib/spinup.asm_riscv64
@@ -6,11 +6,19 @@ riscv_spinup:
.option norelax
csrci sstatus, 0x2
csrw sie, zero
+
+ lla t0, 0f
+ add t0, t0, a3
+ csrw stvec, t0
+ csrw satp, a2
+ sfence.vma
+ unimp
+.align 4
+0:
csrw stvec, zero
mv t0, a0
mv sp, a1
- csrw satp, a2
mv a0, zero
mv a1, zero
diff --git a/common/protos/limine.c b/common/protos/limine.c
index 3f352294..d6de2c08 100644
--- a/common/protos/limine.c
+++ b/common/protos/limine.c
@@ -62,33 +62,8 @@ static pagemap_t build_pagemap(int paging_mode, bool nx, struct elf_range *range
}
}
- // Sub 2MiB mappings
- for (uint64_t i = 0; i < 0x200000; i += 0x1000) {
- if (i != 0) {
- map_page(pagemap, i, i, VMM_FLAG_WRITE, Size4KiB);
- }
- map_page(pagemap, direct_map_offset + i, i, VMM_FLAG_WRITE, Size4KiB);
- }
-
- // Map 2MiB to 4GiB at higher half base and 0
- //
- // NOTE: We cannot just directly map from 2MiB to 4GiB with 1GiB
- // pages because if you do the math.
- //
- // start = 0x200000
- // end = 0x40000000
- //
- // pages_required = (end - start) / (4096 * 512 * 512)
- //
- // So we map 2MiB to 1GiB with 2MiB pages and then map the rest
- // with 1GiB pages :^)
- for (uint64_t i = 0x200000; i < 0x40000000; i += 0x200000) {
- map_page(pagemap, i, i, VMM_FLAG_WRITE, Size2MiB);
- map_page(pagemap, direct_map_offset + i, i, VMM_FLAG_WRITE, Size2MiB);
- }
-
- for (uint64_t i = 0x40000000; i < 0x100000000; i += 0x40000000) {
- map_page(pagemap, i, i, VMM_FLAG_WRITE, Size1GiB);
+ // Map 0->4GiB range to HHDM
+ for (uint64_t i = 0; i < 0x100000000; i += 0x40000000) {
map_page(pagemap, direct_map_offset + i, i, VMM_FLAG_WRITE, Size1GiB);
}
@@ -98,8 +73,14 @@ static pagemap_t build_pagemap(int paging_mode, bool nx, struct elf_range *range
for (size_t i = 0; i < _memmap_entries; i++)
_memmap[i] = memmap[i];
- // Map any other region of memory from the memmap
+ // Map all free memory regions to the higher half direct map offset
for (size_t i = 0; i < _memmap_entries; i++) {
+ if (_memmap[i].type != MEMMAP_USABLE
+ && _memmap[i].type != MEMMAP_BOOTLOADER_RECLAIMABLE
+ && _memmap[i].type != MEMMAP_KERNEL_AND_MODULES) {
+ continue;
+ }
+
uint64_t base = _memmap[i].base;
uint64_t length = _memmap[i].length;
uint64_t top = base + length;
@@ -118,7 +99,6 @@ static pagemap_t build_pagemap(int paging_mode, bool nx, struct elf_range *range
for (uint64_t j = 0; j < aligned_length; j += 0x40000000) {
uint64_t page = aligned_base + j;
- map_page(pagemap, page, page, VMM_FLAG_WRITE, Size1GiB);
map_page(pagemap, direct_map_offset + page, page, VMM_FLAG_WRITE, Size1GiB);
}
}
@@ -139,11 +119,17 @@ static pagemap_t build_pagemap(int paging_mode, bool nx, struct elf_range *range
for (uint64_t j = 0; j < aligned_length; j += 0x1000) {
uint64_t page = aligned_base + j;
- map_page(pagemap, page, page, VMM_FLAG_WRITE | VMM_FLAG_FB, Size4KiB);
map_page(pagemap, direct_map_offset + page, page, VMM_FLAG_WRITE | VMM_FLAG_FB, Size4KiB);
}
}
+ // XXX we do this as a quick and dirty way to switch to the higher half
+#if defined (__x86_64__) || defined (__i386__)
+ for (uint64_t i = 0; i < 0x100000000; i += 0x40000000) {
+ map_page(pagemap, i, i, VMM_FLAG_WRITE, Size1GiB);
+ }
+#endif
+
return pagemap;
}
@@ -979,9 +965,10 @@ FEAT_START
uint64_t bsp_mpidr;
smp_info = init_smp(&cpu_count, &bsp_mpidr,
- pagemap, LIMINE_MAIR(fb_attr), LIMINE_TCR(tsz, pa), LIMINE_SCTLR);
+ pagemap, LIMINE_MAIR(fb_attr), LIMINE_TCR(tsz, pa), LIMINE_SCTLR,
+ direct_map_offset);
#elif defined (__riscv64)
- smp_info = init_smp(&cpu_count, pagemap);
+ smp_info = init_smp(&cpu_count, pagemap, direct_map_offset);
#else
#error Unknown architecture
#endif
@@ -1115,11 +1102,12 @@ FEAT_END
uint64_t reported_stack = reported_addr(stack);
- common_spinup(limine_spinup_32, 8,
+ common_spinup(limine_spinup_32, 10,
paging_mode, (uint32_t)(uintptr_t)pagemap.top_level,
(uint32_t)entry_point, (uint32_t)(entry_point >> 32),
(uint32_t)reported_stack, (uint32_t)(reported_stack >> 32),
- (uint32_t)(uintptr_t)local_gdt, nx_available);
+ (uint32_t)(uintptr_t)local_gdt, nx_available,
+ (uint32_t)direct_map_offset, (uint32_t)(direct_map_offset >> 32));
#elif defined (__aarch64__)
vmm_assert_4k_pages();
@@ -1127,12 +1115,13 @@ FEAT_END
enter_in_el1(entry_point, reported_stack, LIMINE_SCTLR, LIMINE_MAIR(fb_attr), LIMINE_TCR(tsz, pa),
(uint64_t)pagemap.top_level[0],
- (uint64_t)pagemap.top_level[1], 0);
+ (uint64_t)pagemap.top_level[1],
+ direct_map_offset);
#elif defined (__riscv64)
uint64_t reported_stack = reported_addr(stack);
uint64_t satp = make_satp(pagemap.paging_mode, pagemap.top_level);
- riscv_spinup(entry_point, reported_stack, satp);
+ riscv_spinup(entry_point, reported_stack, satp, direct_map_offset);
#else
#error Unknown architecture
#endif
diff --git a/common/protos/limine_32.asm_x86 b/common/protos/limine_32.asm_x86
index 59555493..3b8a7f04 100644
--- a/common/protos/limine_32.asm_x86
+++ b/common/protos/limine_32.asm_x86
@@ -67,6 +67,24 @@ bits 64
mov eax, [rsp+28] ; local_gdt
lgdt [rax]
+ ; Jump to higher half
+ mov rax, qword [rsp+36]
+ add rsp, rax
+ call .p2
+ .p2:
+ add qword [rsp], .hh - .p2
+ add qword [rsp], rax
+ retq
+ .hh:
+
+ ; Unmap lower half entirely
+ mov rsi, cr3
+ lea rdi, [rsi + rax]
+ mov rcx, 256
+ xor rax, rax
+ rep stosq
+ mov cr3, rsi
+
; Push fake return address
mov rsi, [rsp+20] ; stack
sub rsi, 8
diff --git a/common/sys/smp.c b/common/sys/smp.c
index e81a46cc..2467e598 100644
--- a/common/sys/smp.c
+++ b/common/sys/smp.c
@@ -257,6 +257,8 @@ struct limine_smp_info *init_smp(size_t *cpu_count,
struct trampoline_passed_info {
uint64_t smp_tpl_booted_flag;
+ uint64_t smp_tpl_hhdm_offset;
+
uint64_t smp_tpl_ttbr0;
uint64_t smp_tpl_ttbr1;
@@ -279,7 +281,8 @@ static uint32_t psci_cpu_on = 0xC4000003;
static bool try_start_ap(int boot_method, uint64_t method_ptr,
struct limine_smp_info *info_struct,
uint64_t ttbr0, uint64_t ttbr1, uint64_t mair,
- uint64_t tcr, uint64_t sctlr) {
+ uint64_t tcr, uint64_t sctlr,
+ uint64_t hhdm_offset) {
// Prepare the trampoline
static void *trampoline = NULL;
if (trampoline == NULL) {
@@ -301,6 +304,7 @@ static bool try_start_ap(int boot_method, uint64_t method_ptr,
passed_info->smp_tpl_mair = mair;
passed_info->smp_tpl_tcr = tcr;
passed_info->smp_tpl_sctlr = sctlr;
+ passed_info->smp_tpl_hhdm_offset = hhdm_offset;
// Cache coherency between the I-Cache and D-Cache is not guaranteed by the
// architecture and as such we must perform I-Cache invalidation.
@@ -384,7 +388,8 @@ static struct limine_smp_info *try_acpi_smp(size_t *cpu_count,
pagemap_t pagemap,
uint64_t mair,
uint64_t tcr,
- uint64_t sctlr) {
+ uint64_t sctlr,
+ uint64_t hhdm_offset) {
int boot_method = BOOT_WITH_ACPI_PARK;
// Search for FADT table
@@ -472,7 +477,7 @@ static struct limine_smp_info *try_acpi_smp(size_t *cpu_count,
if (!try_start_ap(boot_method, gicc->parking_addr, info_struct,
(uint64_t)(uintptr_t)pagemap.top_level[0],
(uint64_t)(uintptr_t)pagemap.top_level[1],
- mair, tcr, sctlr)) {
+ mair, tcr, sctlr, hhdm_offset)) {
print("smp: FAILED to bring-up AP\n");
continue;
}
@@ -493,16 +498,18 @@ struct limine_smp_info *init_smp(size_t *cpu_count,
pagemap_t pagemap,
uint64_t mair,
uint64_t tcr,
- uint64_t sctlr) {
+ uint64_t sctlr,
+ uint64_t hhdm_offset) {
struct limine_smp_info *info = NULL;
//if (dtb_is_present() && (info = try_dtb_smp(cpu_count,
- // _bsp_iface_no, pagemap, mair, tcr, sctlr)))
+ // _bsp_iface_no, pagemap, mair, tcr, sctlr, hhdm_offset)))
// return info;
// No RSDP means no ACPI
- if (acpi_get_rsdp() && (info = try_acpi_smp(cpu_count,
- bsp_mpidr, pagemap, mair, tcr, sctlr)))
+ if (acpi_get_rsdp() && (info = try_acpi_smp(
+ cpu_count, bsp_mpidr, pagemap,
+ mair, tcr, sctlr, hhdm_offset)))
return info;
printv("Failed to figure out how to start APs.");
@@ -516,14 +523,17 @@ struct trampoline_passed_info {
uint64_t smp_tpl_booted_flag;
uint64_t smp_tpl_satp;
uint64_t smp_tpl_info_struct;
+ uint64_t smp_tpl_hhdm_offset;
};
-static bool smp_start_ap(size_t hartid, size_t satp, struct limine_smp_info *info_struct) {
+static bool smp_start_ap(size_t hartid, size_t satp, struct limine_smp_info *info_struct,
+ uint64_t hhdm_offset) {
static struct trampoline_passed_info passed_info;
passed_info.smp_tpl_booted_flag = 0;
passed_info.smp_tpl_satp = satp;
passed_info.smp_tpl_info_struct = (uint64_t)info_struct;
+ passed_info.smp_tpl_hhdm_offset = hhdm_offset;
asm volatile ("" ::: "memory");
@@ -539,7 +549,7 @@ static bool smp_start_ap(size_t hartid, size_t satp, struct limine_smp_info *inf
return false;
}
-struct limine_smp_info *init_smp(size_t *cpu_count, pagemap_t pagemap) {
+struct limine_smp_info *init_smp(size_t *cpu_count, pagemap_t pagemap, uint64_t hhdm_offset) {
size_t num_cpus = 0;
for (struct riscv_hart *hart = hart_list; hart != NULL; hart = hart->next) {
if (!(hart->flags & RISCV_HART_COPROC)) {
@@ -572,7 +582,7 @@ struct limine_smp_info *init_smp(size_t *cpu_count, pagemap_t pagemap) {
// Try to start the AP.
size_t satp = make_satp(pagemap.paging_mode, pagemap.top_level);
- if (!smp_start_ap(hart->hartid, satp, info_struct)) {
+ if (!smp_start_ap(hart->hartid, satp, info_struct, hhdm_offset)) {
print("smp: FAILED to bring-up AP\n");
continue;
}
diff --git a/common/sys/smp.h b/common/sys/smp.h
index 77a64235..25387fd7 100644
--- a/common/sys/smp.h
+++ b/common/sys/smp.h
@@ -26,12 +26,14 @@ struct limine_smp_info *init_smp(size_t *cpu_count,
pagemap_t pagemap,
uint64_t mair,
uint64_t tcr,
- uint64_t sctlr);
+ uint64_t sctlr,
+ uint64_t hhdm_offset);
#elif defined (__riscv64)
struct limine_smp_info *init_smp(size_t *cpu_count,
- pagemap_t pagemap);
+ pagemap_t pagemap,
+ uint64_t hhdm_offset);
#else
#error Unknown architecture
diff --git a/common/sys/smp_trampoline.asm_aarch64 b/common/sys/smp_trampoline.asm_aarch64
index 149019a1..d65c9db2 100644
--- a/common/sys/smp_trampoline.asm_aarch64
+++ b/common/sys/smp_trampoline.asm_aarch64
@@ -1,6 +1,7 @@
#include <lib/macros.aarch64_asm.h>
-.set tpl_booted_flag, -56
+.set tpl_booted_flag, -64
+.set tpl_hhdm_offset, -56
.set tpl_ttbr0, -48
.set tpl_ttbr1, -40
.set tpl_mair, -32
@@ -26,20 +27,22 @@ smp_trampoline_start:
ldr x4, [x1, tpl_tcr]
ldr x5, [x1, tpl_ttbr0]
ldr x6, [x1, tpl_ttbr1]
+ ldr x7, [x1, tpl_hhdm_offset]
- // Configure EL1 state
+ PICK_EL x8, 1f, 0f
+0:
+ // Configure EL2-specific state for EL1
+
+ // Configure EL1 page tables
msr mair_el1, x3
msr tcr_el1, x4
msr ttbr0_el1, x5
msr ttbr1_el1, x6
msr sctlr_el1, x2
+ isb
dsb sy
isb
- PICK_EL x8, 1f, 0f
-0:
- // Configure EL2-specific state for EL1
-
// Don't trap counters to EL2
mrs x8, cnthctl_el2
orr x8, x8, #3
@@ -60,26 +63,67 @@ smp_trampoline_start:
// Run rest of trampoline in EL1
mov x8, #0x3c4
msr spsr_el2, x8
- adr x8, 1f
+ adrp x8, 3f
+ add x8, x8, :lo12:2f
+ add x8, x8, x7 // Add HHDM offset
msr elr_el2, x8
eret
1:
+ msr spsel, #0
+
+ // Switch to the new page tables
+
+ // Point the EL1t handler to the continuation, such that after we page fault,
+ // execution continues as expected.
+ adrp x8, 2f
+ add x8, x8, #:lo12:2f
+ add x8, x8, x7
+ msr vbar_el1, x8
+ isb
+ dsb sy
+ isb
+
+ // Switch the page table registers
+ msr mair_el1, x3
+ msr tcr_el1, x4
+ msr ttbr0_el1, x5
+ msr ttbr1_el1, x6
+ msr sctlr_el1, x2
+ isb
+ dsb sy
+ isb
+
+ // Jump to the higher half mapping in case we didn't immediately crash
+ br x8
+
+// Alignment required by VBAR register
+.align 11
+2:
+ // Zero out VBAR to avoid confusion
+ msr vbar_el1, xzr
+
+3:
+ // Add HHDM offset to data pointer
+ add x1, x1, x7
+
// Notify BSP we are alive
mov x8, #1
add x9, x1, tpl_booted_flag
stlr x8, [x9]
// Wait for BSP to tell us where to go
+ // Add HHDM offset to our info struct pointer
+ add x0, x0, x7
add x9, x0, #24
-2:
+4:
ldar x8, [x9]
- cbnz x8, 3f
+ cbnz x8, 5f
yield
- b 2b
+ b 4b
-3:
+5:
msr elr_el1, x8
msr spsel, #0
diff --git a/common/sys/smp_trampoline.asm_riscv64 b/common/sys/smp_trampoline.asm_riscv64
index 98f09824..c2749c34 100644
--- a/common/sys/smp_trampoline.asm_riscv64
+++ b/common/sys/smp_trampoline.asm_riscv64
@@ -11,9 +11,26 @@ smp_trampoline_start:
//
// All other registers are undefined.
- ld a0, 16(a1)
- ld t0, 8(a1)
+#define smp_tpl_booted_flag 0
+#define smp_tpl_satp 8
+#define smp_tpl_info_struct 16
+#define smp_tpl_hhdm_offset 24
+
+ ld a0, smp_tpl_info_struct(a1)
+ ld t1, smp_tpl_hhdm_offset(a1)
+
+ // Set `stvec` so we page fault into the higher half after loading `satp`.
+ lla t0, 0f
+ add t0, t1, t0
+ csrw stvec, t0
+ ld t0, smp_tpl_satp(a1)
csrw satp, t0
+ sfence.vma
+ unimp
+0:
+ // Relocate the smp_info and passed_info pointers to the higher half.
+ add a0, t1, a0
+ add a1, t1, a1
// Tell the BSP we've started.
li t0, 1
diff --git a/common/sys/smp_trampoline.asm_x86 b/common/sys/smp_trampoline.asm_x86
index f00b2742..29b13a0f 100644
--- a/common/sys/smp_trampoline.asm_x86
+++ b/common/sys/smp_trampoline.asm_x86
@@ -137,6 +137,10 @@ parking64:
jmp .loop
.out:
+ ; Clear TLB
+ mov rbx, cr3
+ mov cr3, rbx
+
mov rsp, qword [rdi + 8]
push 0
push rax
