mm/efi_pt: Retag firmware FB PTEs to UC- so WC MTRRs take effect
diff --git a/common/lib/gterm.c b/common/lib/gterm.c
index 0e59f514..2cded5ee 100644
--- a/common/lib/gterm.c
+++ b/common/lib/gterm.c
@@ -11,6 +11,7 @@
#include <lib/rand.h>
#include <mm/pmm.h>
#include <mm/mtrr.h>
+#include <mm/efi_pt.h>
#include <flanterm.h>
#include <flanterm_backends/fb.h>
#include <lib/term.h>
@@ -820,6 +821,9 @@ bool gterm_init(struct fb_info **_fbs, size_t *_fbs_count,
continue;
}
mtrr_wc_add_fb_range(fbs[i].framebuffer_addr, fb_size);
+#if defined (__x86_64__) && defined (UEFI)
+ efi_pt_set_fb_uc_minus(fbs[i].framebuffer_addr, fb_size);
+#endif
}
#endif
diff --git a/common/mm/efi_pt.c b/common/mm/efi_pt.c
new file mode 100644
index 00000000..4285fddb
--- /dev/null
+++ b/common/mm/efi_pt.c
@@ -0,0 +1,107 @@
+#if defined (__x86_64__) && defined (UEFI)
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <mm/efi_pt.h>
+#include <sys/cpu.h>
+
+#define PTE_P ((uint64_t)1 << 0)
+#define PTE_PWT ((uint64_t)1 << 3)
+#define PTE_PCD ((uint64_t)1 << 4)
+#define PTE_PS ((uint64_t)1 << 7)
+#define PTE_PAT_4K ((uint64_t)1 << 7)
+#define PTE_PAT_BIG ((uint64_t)1 << 12)
+#define PT_ADDR_MASK ((uint64_t)0x000FFFFFFFFFF000)
+
+#define UCM_MASK_4K (PTE_PWT | PTE_PCD | PTE_PAT_4K)
+#define UCM_MASK_BIG (PTE_PWT | PTE_PCD | PTE_PAT_BIG)
+#define UCM_VALUE PTE_PCD
+
+#define IA32_PAT_MSR 0x277
+#define PAT_TYPE_UCM 0x07
+
+static bool la57_enabled(void) {
+ uint64_t cr4;
+ asm volatile ("mov %%cr4, %0" : "=r"(cr4));
+ return !!(cr4 & ((uint64_t)1 << 12));
+}
+
+static bool pat_slot2_is_ucm(void) {
+ static bool checked = false, ok = false;
+ if (checked) {
+ return ok;
+ }
+ checked = true;
+
+ uint32_t eax, ebx, ecx, edx;
+ if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx) || !(edx & (1 << 16))) {
+ return false;
+ }
+
+ ok = ((rdmsr(IA32_PAT_MSR) >> 16) & 0xff) == PAT_TYPE_UCM;
+ return ok;
+}
+
+static uint64_t *walk_to_leaf(uint64_t addr, uint64_t *pg_size, bool *large) {
+ uint64_t cr3;
+ asm volatile ("mov %%cr3, %0" : "=r"(cr3));
+
+ uint64_t *table = (uint64_t *)(cr3 & PT_ADDR_MASK);
+
+ int lvl = la57_enabled() ? 5 : 4;
+
+ for (; lvl >= 1; lvl--) {
+ int shift = (lvl - 1) * 9 + 12;
+ size_t idx = (addr >> shift) & 0x1ff;
+ uint64_t e = table[idx];
+
+ if (!(e & PTE_P)) {
+ return NULL;
+ }
+
+ bool is_leaf = (lvl == 1) || ((lvl == 2 || lvl == 3) && (e & PTE_PS));
+ if (is_leaf) {
+ *pg_size = (uint64_t)1 << shift;
+ *large = (lvl != 1);
+ return &table[idx];
+ }
+
+ table = (uint64_t *)(e & PT_ADDR_MASK);
+ }
+ return NULL;
+}
+
+void efi_pt_set_fb_uc_minus(uint64_t base, uint64_t size) {
+ if (size == 0 || !pat_slot2_is_ucm()) {
+ return;
+ }
+
+ // Clear CR0.WP so supervisor writes hit any firmware PTEs mapped R/O.
+ uint64_t old_cr0;
+ asm volatile ("mov %%cr0, %0" : "=r"(old_cr0));
+ asm volatile ("mov %0, %%cr0" :: "r"(old_cr0 & ~((uint64_t)1 << 16)) : "memory");
+
+ uint64_t end = (base + size + 0xfff) & ~(uint64_t)0xfff;
+ base &= ~(uint64_t)0xfff;
+
+ while (base < end) {
+ uint64_t pg;
+ bool large;
+ uint64_t *entry = walk_to_leaf(base, &pg, &large);
+ if (entry == NULL) {
+ base += 0x1000;
+ continue;
+ }
+
+ uint64_t mask = large ? UCM_MASK_BIG : UCM_MASK_4K;
+ *entry = (*entry & ~mask) | UCM_VALUE;
+
+ invlpg(base);
+ base = (base & ~(pg - 1)) + pg;
+ }
+
+ asm volatile ("mov %0, %%cr0" :: "r"(old_cr0) : "memory");
+}
+
+#endif
diff --git a/common/mm/efi_pt.h b/common/mm/efi_pt.h
new file mode 100644
index 00000000..ee6c57f6
--- /dev/null
+++ b/common/mm/efi_pt.h
@@ -0,0 +1,12 @@
+#ifndef MM__EFI_PT_H__
+#define MM__EFI_PT_H__
+
+#include <stdint.h>
+
+#if defined (__x86_64__) && defined (UEFI)
+
+void efi_pt_set_fb_uc_minus(uint64_t base, uint64_t size);
+
+#endif
+
+#endif
diff --git a/common/sys/cpu.h b/common/sys/cpu.h
index a91ed5e2..6dcb385c 100644
--- a/common/sys/cpu.h
+++ b/common/sys/cpu.h
@@ -160,6 +160,10 @@ static inline uint64_t rdtsc(void) {
return ((uint64_t)edx << 32) | eax;
}
+static inline void invlpg(uintptr_t va) {
+ asm volatile ("invlpg (%0)" :: "r"(va) : "memory");
+}
+
static inline uint64_t tsc_freq_arch(void) {
uint32_t eax, ebx, ecx, edx;
if (!cpuid(0x15, 0, &eax, &ebx, &ecx, &edx))
