| 1 | #if defined (__x86_64__) || defined (__i386__) |
| 2 | |
| 3 | #include <stdint.h> |
| 4 | #include <stddef.h> |
| 5 | #include <mm/mtrr.h> |
| 6 | #include <mm/pmm.h> |
| 7 | #include <sys/cpu.h> |
| 8 | #include <lib/print.h> |
| 9 | #include <lib/misc.h> |
| 10 | |
| 11 | static bool mtrr_supported(void) { |
| 12 | uint32_t eax, ebx, ecx, edx; |
| 13 | |
| 14 | if (!cpuid(1, 0, &eax, &ebx, &ecx, &edx)) |
| 15 | return false; |
| 16 | |
| 17 | return !!(edx & (1 << 12)); |
| 18 | } |
| 19 | |
| 20 | uint64_t *saved_mtrrs = NULL; |
| 21 | |
| 22 | void mtrr_save(void) { |
| 23 | if (saved_mtrrs != NULL) { |
| 24 | return; |
| 25 | } |
| 26 | if (!mtrr_supported()) { |
| 27 | return; |
| 28 | } |
| 29 | |
| 30 | bool ints = disable_interrupts(); |
| 31 | |
| 32 | uint64_t ia32_mtrrcap = rdmsr(0xfe); |
| 33 | |
| 34 | uint8_t var_reg_count = ia32_mtrrcap & 0xff; |
| 35 | bool fix_supported = !!(ia32_mtrrcap & ((uint64_t)1 << 8)); |
| 36 | |
| 37 | saved_mtrrs = ext_mem_alloc(( |
| 38 | (var_reg_count * 2) /* variable MTRRs, 2 MSRs each */ |
| 39 | + 11 /* 11 fixed MTRRs */ |
| 40 | + 1 /* 1 default type MTRR */ |
| 41 | ) * sizeof(uint64_t)); |
| 42 | |
| 43 | /* save variable range MTRRs */ |
| 44 | for (uint8_t i = 0; i < var_reg_count * 2; i += 2) { |
| 45 | saved_mtrrs[i] = rdmsr(0x200 + i); |
| 46 | saved_mtrrs[i + 1] = rdmsr(0x200 + i + 1); |
| 47 | } |
| 48 | |
| 49 | /* save fixed range MTRRs, if supported by the CPU */ |
| 50 | if (fix_supported) { |
| 51 | saved_mtrrs[var_reg_count * 2 + 0] = rdmsr(0x250); |
| 52 | saved_mtrrs[var_reg_count * 2 + 1] = rdmsr(0x258); |
| 53 | saved_mtrrs[var_reg_count * 2 + 2] = rdmsr(0x259); |
| 54 | saved_mtrrs[var_reg_count * 2 + 3] = rdmsr(0x268); |
| 55 | saved_mtrrs[var_reg_count * 2 + 4] = rdmsr(0x269); |
| 56 | saved_mtrrs[var_reg_count * 2 + 5] = rdmsr(0x26a); |
| 57 | saved_mtrrs[var_reg_count * 2 + 6] = rdmsr(0x26b); |
| 58 | saved_mtrrs[var_reg_count * 2 + 7] = rdmsr(0x26c); |
| 59 | saved_mtrrs[var_reg_count * 2 + 8] = rdmsr(0x26d); |
| 60 | saved_mtrrs[var_reg_count * 2 + 9] = rdmsr(0x26e); |
| 61 | saved_mtrrs[var_reg_count * 2 + 10] = rdmsr(0x26f); |
| 62 | } |
| 63 | |
| 64 | /* save MTRR default type */ |
| 65 | saved_mtrrs[var_reg_count * 2 + 11] = rdmsr(0x2ff); |
| 66 | |
| 67 | /* make sure that the saved MTRR default has MTRRs off */ |
| 68 | saved_mtrrs[var_reg_count * 2 + 11] &= ~((uint64_t)1 << 11); |
| 69 | |
| 70 | if (ints) { |
| 71 | enable_interrupts(); |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | void mtrr_restore(void) { |
| 76 | if (!mtrr_supported()) { |
| 77 | return; |
| 78 | } |
| 79 | |
| 80 | uint64_t ia32_mtrrcap = rdmsr(0xfe); |
| 81 | |
| 82 | uint8_t var_reg_count = ia32_mtrrcap & 0xff; |
| 83 | bool fix_supported = !!(ia32_mtrrcap & ((uint64_t)1 << 8)); |
| 84 | |
| 85 | if (saved_mtrrs == NULL) { |
| 86 | return; |
| 87 | } |
| 88 | |
| 89 | bool ints = disable_interrupts(); |
| 90 | |
| 91 | /* according to the Intel SDM 12.11.7.2 "MemTypeSet() Function", |
| 92 | we need to follow this procedure before changing MTRR set up */ |
| 93 | |
| 94 | /* save old cr0 and then enable the CD flag and disable the NW flag */ |
| 95 | uintptr_t old_cr0; |
| 96 | asm volatile ("mov %%cr0, %0" : "=r"(old_cr0) :: "memory"); |
| 97 | uintptr_t new_cr0 = (old_cr0 | (1 << 30)) & ~((uintptr_t)1 << 29); |
| 98 | asm volatile ("mov %0, %%cr0" :: "r"(new_cr0) : "memory"); |
| 99 | |
| 100 | /* then invalidate the caches */ |
| 101 | asm volatile ("wbinvd" ::: "memory"); |
| 102 | |
| 103 | /* do a cr3 read/write to flush the TLB */ |
| 104 | uintptr_t cr3; |
| 105 | asm volatile ("mov %%cr3, %0" : "=r"(cr3) :: "memory"); |
| 106 | asm volatile ("mov %0, %%cr3" :: "r"(cr3) : "memory"); |
| 107 | |
| 108 | /* disable the MTRRs */ |
| 109 | uint64_t mtrr_def = rdmsr(0x2ff); |
| 110 | mtrr_def &= ~((uint64_t)1 << 11); |
| 111 | wrmsr(0x2ff, mtrr_def); |
| 112 | |
| 113 | /* restore variable range MTRRs */ |
| 114 | for (uint8_t i = 0; i < var_reg_count * 2; i += 2) { |
| 115 | wrmsr(0x200 + i, saved_mtrrs[i]); |
| 116 | wrmsr(0x200 + i + 1, saved_mtrrs[i + 1]); |
| 117 | } |
| 118 | |
| 119 | /* restore fixed range MTRRs, if supported by the CPU */ |
| 120 | if (fix_supported) { |
| 121 | wrmsr(0x250, saved_mtrrs[var_reg_count * 2 + 0]); |
| 122 | wrmsr(0x258, saved_mtrrs[var_reg_count * 2 + 1]); |
| 123 | wrmsr(0x259, saved_mtrrs[var_reg_count * 2 + 2]); |
| 124 | wrmsr(0x268, saved_mtrrs[var_reg_count * 2 + 3]); |
| 125 | wrmsr(0x269, saved_mtrrs[var_reg_count * 2 + 4]); |
| 126 | wrmsr(0x26a, saved_mtrrs[var_reg_count * 2 + 5]); |
| 127 | wrmsr(0x26b, saved_mtrrs[var_reg_count * 2 + 6]); |
| 128 | wrmsr(0x26c, saved_mtrrs[var_reg_count * 2 + 7]); |
| 129 | wrmsr(0x26d, saved_mtrrs[var_reg_count * 2 + 8]); |
| 130 | wrmsr(0x26e, saved_mtrrs[var_reg_count * 2 + 9]); |
| 131 | wrmsr(0x26f, saved_mtrrs[var_reg_count * 2 + 10]); |
| 132 | } |
| 133 | |
| 134 | /* restore MTRR default type */ |
| 135 | wrmsr(0x2ff, saved_mtrrs[var_reg_count * 2 + 11]); |
| 136 | |
| 137 | /* now do the opposite of the cache disable and flush from above */ |
| 138 | |
| 139 | /* re-enable MTRRs */ |
| 140 | mtrr_def = rdmsr(0x2ff); |
| 141 | mtrr_def |= (1 << 11); |
| 142 | wrmsr(0x2ff, mtrr_def); |
| 143 | |
| 144 | /* do a cr3 read/write to flush the TLB */ |
| 145 | asm volatile ("mov %%cr3, %0" : "=r"(cr3) :: "memory"); |
| 146 | asm volatile ("mov %0, %%cr3" :: "r"(cr3) : "memory"); |
| 147 | |
| 148 | /* then invalidate the caches */ |
| 149 | asm volatile ("wbinvd" ::: "memory"); |
| 150 | |
| 151 | /* restore old value of cr0 */ |
| 152 | asm volatile ("mov %0, %%cr0" :: "r"(old_cr0) : "memory"); |
| 153 | |
| 154 | if (ints) { |
| 155 | enable_interrupts(); |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | #define MTRR_TYPE_WC 1 |
| 160 | |
| 161 | static bool wc_add_mtrr(uint64_t base, uint64_t size, uint8_t type, |
| 162 | uint8_t maxphysaddr, uint8_t var_reg_count) { |
| 163 | uint8_t slot = 0xff; |
| 164 | for (uint8_t i = 0; i < var_reg_count; i++) { |
| 165 | if (!(rdmsr(0x200 + i * 2 + 1) & ((uint64_t)1 << 11))) { |
| 166 | slot = i; |
| 167 | break; |
| 168 | } |
| 169 | } |
| 170 | if (slot == 0xff) { |
| 171 | return false; |
| 172 | } |
| 173 | uint64_t mask = (((uint64_t)1 << maxphysaddr) - 1) & ~(size - 1); |
| 174 | wrmsr(0x200 + slot * 2, base | type); |
| 175 | wrmsr(0x200 + slot * 2 + 1, mask | ((uint64_t)1 << 11)); |
| 176 | return true; |
| 177 | } |
| 178 | |
| 179 | static bool wc_add_chunks(uint64_t base, uint64_t span, uint8_t type, |
| 180 | uint8_t maxphysaddr, uint8_t var_reg_count) { |
| 181 | while (span > 0) { |
| 182 | uint64_t align_k = base != 0 ? ((uint64_t)1 << __builtin_ctzll(base)) : span; |
| 183 | uint64_t size_k = (uint64_t)1 << (63 - __builtin_clzll(span)); |
| 184 | uint64_t k = align_k < size_k ? align_k : size_k; |
| 185 | |
| 186 | if (!wc_add_mtrr(base, k, type, maxphysaddr, var_reg_count)) { |
| 187 | return false; |
| 188 | } |
| 189 | base += k; |
| 190 | span -= k; |
| 191 | } |
| 192 | return true; |
| 193 | } |
| 194 | |
| 195 | bool mtrr_wc_add_fb_range(uint64_t base, uint64_t size) { |
| 196 | if (size == 0 || !mtrr_supported()) { |
| 197 | return false; |
| 198 | } |
| 199 | |
| 200 | uint64_t mtrrcap = rdmsr(0xfe); |
| 201 | if (!(mtrrcap & ((uint64_t)1 << 10))) { |
| 202 | return false; |
| 203 | } |
| 204 | uint8_t var_reg_count = mtrrcap & 0xff; |
| 205 | |
| 206 | uint32_t eax, ebx, ecx, edx; |
| 207 | if (!cpuid(0x80000008, 0, &eax, &ebx, &ecx, &edx)) { |
| 208 | return false; |
| 209 | } |
| 210 | uint8_t maxphysaddr = eax & 0xff; |
| 211 | if (maxphysaddr < 32 || maxphysaddr > 52) { |
| 212 | return false; |
| 213 | } |
| 214 | |
| 215 | uint64_t end = (base + size + 0xfff) & ~(uint64_t)0xfff; |
| 216 | base &= ~(uint64_t)0xfff; |
| 217 | size = end - base; |
| 218 | |
| 219 | for (uint8_t i = 0; i < var_reg_count; i++) { |
| 220 | uint64_t mb = rdmsr(0x200 + i * 2); |
| 221 | uint64_t mm = rdmsr(0x200 + i * 2 + 1); |
| 222 | if (!(mm & ((uint64_t)1 << 11))) { |
| 223 | continue; |
| 224 | } |
| 225 | uint64_t exist_mask = mm & ~(uint64_t)0xfff; |
| 226 | uint64_t exist_base = mb & ~(uint64_t)0xfff; |
| 227 | for (uint64_t a = base; a < end; a += 0x1000) { |
| 228 | if ((a & exist_mask) == (exist_base & exist_mask)) { |
| 229 | return false; |
| 230 | } |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | mtrr_save(); |
| 235 | |
| 236 | #if defined (UEFI) |
| 237 | asm volatile ("cli"); |
| 238 | #endif |
| 239 | |
| 240 | uintptr_t old_cr0; |
| 241 | asm volatile ("mov %%cr0, %0" : "=r"(old_cr0) :: "memory"); |
| 242 | uintptr_t new_cr0 = (old_cr0 | (1U << 30)) & ~((uintptr_t)1 << 29); |
| 243 | asm volatile ("mov %0, %%cr0" :: "r"(new_cr0) : "memory"); |
| 244 | asm volatile ("wbinvd" ::: "memory"); |
| 245 | |
| 246 | uintptr_t cr3; |
| 247 | asm volatile ("mov %%cr3, %0" : "=r"(cr3) :: "memory"); |
| 248 | asm volatile ("mov %0, %%cr3" :: "r"(cr3) : "memory"); |
| 249 | |
| 250 | uint64_t mtrr_def = rdmsr(0x2ff); |
| 251 | wrmsr(0x2ff, mtrr_def & ~((uint64_t)1 << 11)); |
| 252 | |
| 253 | bool ok = wc_add_chunks(base, size, MTRR_TYPE_WC, maxphysaddr, var_reg_count); |
| 254 | |
| 255 | wrmsr(0x2ff, mtrr_def); |
| 256 | |
| 257 | asm volatile ("mov %%cr3, %0" : "=r"(cr3) :: "memory"); |
| 258 | asm volatile ("mov %0, %%cr3" :: "r"(cr3) : "memory"); |
| 259 | asm volatile ("wbinvd" ::: "memory"); |
| 260 | asm volatile ("mov %0, %%cr0" :: "r"(old_cr0) : "memory"); |
| 261 | |
| 262 | #if defined (UEFI) |
| 263 | asm volatile ("sti"); |
| 264 | #endif |
| 265 | |
| 266 | return ok; |
| 267 | } |
| 268 | |
| 269 | #endif |
