misc.c 13.7 KB raw

#include <stdint.h>
#include <stddef.h>
#include <stdarg.h>
#include <lib/libc.h>
#include <lib/misc.h>
#include <lib/print.h>
#include <lib/trace.h>
#include <lib/real.h>
#include <lib/config.h>
#include <lib/uri.h>
#include <lib/bli.h>
#include <lib/rng_seed.h>
#include <lib/tpm.h>
#include <fs/file.h>
#include <mm/pmm.h>
#include <libfdt.h>
​
#if defined (UEFI)
EFI_SYSTEM_TABLE *gST;
EFI_BOOT_SERVICES *gBS;
EFI_RUNTIME_SERVICES *gRT;
EFI_HANDLE efi_image_handle;
EFI_MEMORY_DESCRIPTOR *efi_mmap = NULL;
UINTN efi_mmap_size = 0, efi_desc_size = 0, efi_mmap_key = 0;
UINT32 efi_desc_ver = 0;
#endif
​
bool editor_enabled = true;
bool help_hidden = false;
bool secure_boot_active = false;
​
uint64_t usec_at_bootloader_entry;
​
#if defined (UEFI)
bool is_efi_serial_present(void) {
    EFI_STATUS status;
    EFI_SERIAL_IO_PROTOCOL *serial_io = NULL;
    EFI_GUID serial_io_guid = EFI_SERIAL_IO_PROTOCOL_GUID;
​
    status = gBS->LocateProtocol(&serial_io_guid, NULL, (void **)&serial_io);
    if (status) {
        return false;
    }
​
    if (serial_io == NULL) {
        return false;
    }
​
    UINT32 control;
    status = serial_io->GetControl(serial_io, &control);
    if (status) {
        return false;
    }
​
    return true;
}
#endif
​
bool parse_resolution(size_t *width, size_t *height, size_t *bpp, const char *buf) {
    size_t res[3] = {0};
​
    const char *first = buf;
    for (size_t i = 0; i < 3; i++) {
        const char *last;
        size_t x = strtoui(first, &last, 10);
        if (first == last)
            break;
        res[i] = x;
        if (*last == 0)
            break;
        first = last + 1;
    }
​
    if (res[0] == 0 || res[1] == 0)
        return false;
​
    if (res[2] == 0)
        res[2] = 32;
​
    *width = res[0], *height = res[1];
    if (bpp != NULL)
        *bpp = res[2];
​
    return true;
}
​
// This integer sqrt implementation has been adapted from:
// https://stackoverflow.com/questions/1100090/looking-for-an-efficient-integer-square-root-algorithm-for-arm-thumb2
uint64_t sqrt(uint64_t a_nInput) {
    uint64_t op  = a_nInput;
    uint64_t res = 0;
    uint64_t one = (uint64_t)1 << 62;
​
    // "one" starts at the highest power of four <= than the argument.
    while (one > op) {
        one >>= 2;
    }
​
    while (one != 0) {
        if (op >= res + one) {
            op = op - (res + one);
            res = res +  2 * one;
        }
        res >>= 1;
        one >>= 2;
    }
​
    return res;
}
​
size_t get_trailing_zeros(uint64_t val) {
    for (size_t i = 0; i < 64; i++) {
        if ((val & 1) != 0) {
            return i;
        }
        val >>= 1;
    }
    return 64;
}
​
void *get_device_tree_blob(const char *config, size_t extra_size,
                           bool measure) {
    int ret;
​
    size_t size = 0;
    void *dtb = NULL;
​
    {
        char *dtb_path = NULL;
        bool soft_panic;
        if (config != NULL) {
            dtb_path = config_get_value(config, 0, "dtb_path");
            soft_panic = true;
        }
        if (dtb_path == NULL) {
            dtb_path = config_get_value(NULL, 0, "global_dtb");
            soft_panic = false;
        }
        if (dtb_path != NULL) {
            struct file_handle *dtb_file;
            if ((dtb_file = uri_open(dtb_path, MEMMAP_BOOTLOADER_RECLAIMABLE, false
#if defined (__i386__)
                , NULL, NULL
#endif
            )) == NULL)
                panic(soft_panic, "dtb: Failed to open device tree blob with path `%#`. Is the path correct?", dtb_path);
​
            dtb = dtb_file->fd;
            size = dtb_file->size;
            fclose(dtb_file);
​
            ret = fdt_check_full(dtb, size);
            if (ret != 0) {
                panic(soft_panic, "dtb: Invalid device tree blob at `%#`: '%s'", dtb_path, fdt_strerror(ret));
            }
​
#if defined (UEFI)
            if (measure) {
                tpm_measure_path(TPM_PCR_BOOT_AUTH, TPM_EV_IPL, "dtb_path: ", dtb_path);
                tpm_measure(TPM_PCR_LOADED_IMAGES, TPM_EV_IPL,
                            dtb, size, "dtb_path: ", dtb_path);
            }
#endif
​
            printv("dtb: loaded dtb at %p from file `%#`\n", dtb, dtb_path);
        }
    }
​
#if defined (UEFI)
    if (!dtb) {
        EFI_GUID dtb_guid = EFI_DTB_TABLE_GUID;
        for (size_t i = 0; i < gST->NumberOfTableEntries; i++) {
            EFI_CONFIGURATION_TABLE *cur_table = &gST->ConfigurationTable[i];
            if (memcmp(&cur_table->VendorGuid, &dtb_guid, sizeof(EFI_GUID)))
                continue;
            size = fdt_totalsize(cur_table->VendorTable);
            if (measure) {
                tpm_measure(TPM_PCR_LOADED_IMAGES, TPM_EV_IPL,
                            cur_table->VendorTable, size, "efi_dtb", NULL);
            }
            dtb = ext_mem_alloc(size);
            ret = fdt_open_into(cur_table->VendorTable, dtb, size);
            if (ret < 0) {
                panic(true, "dtb: failed to resize new DTB");
            }
            printv("dtb: found dtb at %p via EFI\n", cur_table->VendorTable);
            break;
        }
    }
#else
    (void)measure;
#endif
​
    if (extra_size == 0) {
        return dtb;
    }
​
    if (dtb) {
        printv("dtb: dtb has size %X\n", (uint64_t)size);
​
        size_t new_size = CHECKED_ADD(size, extra_size,
            panic(true, "dtb: size overflow"));
        void *new_tab = ext_mem_alloc(new_size);
​
        ret = fdt_open_into(dtb, new_tab, new_size);
        if (ret < 0) {
            panic(true, "dtb: failed to resize new DTB");
        }
​
        pmm_free(dtb, size);
        return new_tab;
    }
​
    dtb = ext_mem_alloc(extra_size);
​
    ret = fdt_create_empty_tree(dtb, extra_size);
    if (ret < 0) {
        panic(true, "dtb: failed to create a device tree blob: '%s'", fdt_strerror(ret));
    }
​
    ret = fdt_setprop_u32(dtb, 0, "#address-cells", 2);
    if (ret < 0) {
        panic(true, "dtb: failed to set #address-cells: '%s'", fdt_strerror(ret));
    }
​
    ret = fdt_setprop_u32(dtb, 0, "#size-cells", 1);
    if (ret < 0) {
        panic(true, "dtb: failed to set #size-cells: '%s'", fdt_strerror(ret));
    }
​
    return dtb;
}
​
#if defined (UEFI)
​
#if defined (__riscv)
​
RISCV_EFI_BOOT_PROTOCOL *get_riscv_boot_protocol(void) {
    EFI_GUID boot_proto_guid = RISCV_EFI_BOOT_PROTOCOL_GUID;
    RISCV_EFI_BOOT_PROTOCOL *proto;
​
    // LocateProtocol() is available from EFI version 1.1
    if (gBS->Hdr.Revision >= ((1 << 16) | 10)) {
        if (gBS->LocateProtocol(&boot_proto_guid, NULL, (void **)&proto) == EFI_SUCCESS) {
            return proto;
        }
    }
​
    UINTN bufsz = 0;
    if (gBS->LocateHandle(ByProtocol, &boot_proto_guid, NULL, &bufsz, NULL) != EFI_BUFFER_TOO_SMALL)
        return NULL;
​
    UINTN handles_alloc = bufsz;
    EFI_HANDLE *handles_buf = ext_mem_alloc(handles_alloc);
    if (handles_buf == NULL)
        return NULL;
​
    if (bufsz < sizeof(EFI_HANDLE))
        goto error;
​
    if (gBS->LocateHandle(ByProtocol, &boot_proto_guid, NULL, &bufsz, handles_buf) != EFI_SUCCESS)
        goto error;
​
    if (gBS->HandleProtocol(handles_buf[0], &boot_proto_guid, (void **)&proto) != EFI_SUCCESS)
        goto error;
​
    pmm_free(handles_buf, handles_alloc);
    return proto;
​
error:
    pmm_free(handles_buf, handles_alloc);
    return NULL;
}
​
#endif
​
no_unwind bool efi_boot_services_exited = false;
​
bool efi_exit_boot_services(void) {
    EFI_STATUS status;
​
    // Pull entropy from EFI_RNG_PROTOCOL while it's still callable and
    // publish it for the kernel to mix into its early RNG state.
    rng_seed_install();
​
    // Free the buffer init_memmap left us; the loop below manages
    // allocation lifetime itself.
    status = gBS->FreePool(efi_mmap);
    if (status) {
        goto fail;
    }
    efi_mmap = NULL;
​
    EFI_MEMORY_DESCRIPTOR *efi_copy = NULL;
    UINTN efi_mmap_alloc = 0;
    UINTN efi_copy_alloc = 0;
​
    bli_on_boot();
​
    for (size_t retries = 0; ; retries++) {
        if (retries == 128) {
            goto fail;
        }
​
        efi_mmap_size = efi_mmap_alloc;
        status = gBS->GetMemoryMap(&efi_mmap_size, efi_mmap, &efi_mmap_key,
                                   &efi_desc_size, &efi_desc_ver);
        if (status == EFI_BUFFER_TOO_SMALL) {
            // Map grew (or first iteration). Free both buffers and
            // reallocate, with slack for the descriptors AllocatePool
            // itself may add.
            if (efi_mmap != NULL) {
                gBS->FreePool(efi_mmap);
                efi_mmap = NULL;
            }
            if (efi_copy != NULL) {
                gBS->FreePool(efi_copy);
                efi_copy = NULL;
            }
            efi_mmap_alloc = efi_mmap_size + 4096;
            status = gBS->AllocatePool(EfiLoaderData, efi_mmap_alloc,
                                       (void **)&efi_mmap);
            if (status) {
                goto fail;
            }
            efi_copy_alloc = CHECKED_MUL(efi_mmap_alloc, (UINTN)2, goto fail);
            status = gBS->AllocatePool(EfiLoaderData, efi_copy_alloc,
                                       (void **)&efi_copy);
            if (status) {
                goto fail;
            }
            continue;
        }
        if (status) {
            goto fail;
        }
​
        // Be gone, UEFI!
        status = gBS->ExitBootServices(efi_image_handle, efi_mmap_key);
        if (status == EFI_SUCCESS) {
            break;
        }
        // Map key invalidated by an allocation - retry.
    }
​
    const size_t EFI_COPY_MAX_ENTRIES = efi_copy_alloc / efi_desc_size;
​
#if defined(__x86_64__) || defined(__i386__)
    asm volatile ("cli" ::: "memory");
#elif defined (__aarch64__)
    asm volatile ("msr daifset, #15" ::: "memory");
#elif defined (__riscv)
    asm volatile ("csrci sstatus, 0x2" ::: "memory");
#elif defined (__loongarch64)
    asm volatile ("csrxchg $r0, %0, 0x0" :: "r" (0x4) : "memory");
#else
#error Unknown architecture
#endif
​
    // Go through new EFI memmap and free up bootloader entries
    size_t entry_count = efi_mmap_size / efi_desc_size;
​
    size_t efi_copy_i = 0;
​
    for (size_t i = 0; i < entry_count; i++) {
        EFI_MEMORY_DESCRIPTOR *orig_entry = (void *)efi_mmap + i * efi_desc_size;
        EFI_MEMORY_DESCRIPTOR *new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size;
​
        if (orig_entry->NumberOfPages == 0) {
            continue;
        }
​
        memcpy(new_entry, orig_entry, efi_desc_size);
​
        uint64_t base = orig_entry->PhysicalStart;
        uint64_t length = orig_entry->NumberOfPages * 4096;
        uint64_t top = base + length;
​
        // Find for a match in the untouched memory map
        for (size_t j = 0; j < untouched_memmap_entries; j++) {
            if (untouched_memmap[j].type != MEMMAP_USABLE)
                continue;
​
            if (top > untouched_memmap[j].base && top <= untouched_memmap[j].base + untouched_memmap[j].length) {
                if (untouched_memmap[j].base < base) {
                    new_entry->NumberOfPages = (base - untouched_memmap[j].base) / 4096;
​
                    efi_copy_i++;
                    if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
                        panic(false, "efi: New memory map exhausted");
                    }
                    new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size;
                    memcpy(new_entry, orig_entry, efi_desc_size);
​
                    new_entry->NumberOfPages -= (base - untouched_memmap[j].base) / 4096;
                    new_entry->PhysicalStart = base;
                    new_entry->VirtualStart = 0;
​
                    length = new_entry->NumberOfPages * 4096;
                    top = base + length;
                }
​
                if (untouched_memmap[j].base > base) {
                    new_entry->NumberOfPages = (untouched_memmap[j].base - base) / 4096;
​
                    efi_copy_i++;
                    if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
                        panic(false, "efi: New memory map exhausted");
                    }
                    new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size;
                    memcpy(new_entry, orig_entry, efi_desc_size);
​
                    new_entry->NumberOfPages -= (untouched_memmap[j].base - base) / 4096;
                    new_entry->PhysicalStart = untouched_memmap[j].base;
                    new_entry->VirtualStart = 0;
​
                    base = new_entry->PhysicalStart;
                    length = new_entry->NumberOfPages * 4096;
                    top = base + length;
                }
​
                if (length < untouched_memmap[j].length) {
                    panic(false, "efi: Memory map corruption");
                }
​
                new_entry->Type = EfiConventionalMemory;
​
                if (length == untouched_memmap[j].length) {
                    // It's a perfect match!
                    break;
                }
​
                new_entry->NumberOfPages = untouched_memmap[j].length / 4096;
​
                efi_copy_i++;
                if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
                    panic(false, "efi: New memory map exhausted");
                }
                new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size;
                memcpy(new_entry, orig_entry, efi_desc_size);
​
                new_entry->NumberOfPages = (length - untouched_memmap[j].length) / 4096;
                new_entry->PhysicalStart = base + untouched_memmap[j].length;
                new_entry->VirtualStart = 0;
​
                break;
            }
        }
​
        efi_copy_i++;
        if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
            panic(false, "efi: New memory map exhausted");
        }
    }
​
    efi_mmap = efi_copy;
    efi_mmap_size = efi_copy_i * efi_desc_size;
​
    efi_boot_services_exited = true;
​
    printv("efi: Exited boot services.\n");
​
    return true;
​
fail:
    panic(false, "efi: Failed to exit boot services");
}
​
#endif

tab: 2 4 8 wrap: off on

1	#include <stdint.h>
2	#include <stddef.h>
3	#include <stdarg.h>
4	#include <lib/libc.h>
5	#include <lib/misc.h>
6	#include <lib/print.h>
7	#include <lib/trace.h>
8	#include <lib/real.h>
9	#include <lib/config.h>
10	#include <lib/uri.h>
11	#include <lib/bli.h>
12	#include <lib/rng_seed.h>
13	#include <lib/tpm.h>
14	#include <fs/file.h>
15	#include <mm/pmm.h>
16	#include <libfdt.h>
17
18	#if defined (UEFI)
19	EFI_SYSTEM_TABLE *gST;
20	EFI_BOOT_SERVICES *gBS;
21	EFI_RUNTIME_SERVICES *gRT;
22	EFI_HANDLE efi_image_handle;
23	EFI_MEMORY_DESCRIPTOR *efi_mmap = NULL;
24	UINTN efi_mmap_size = 0, efi_desc_size = 0, efi_mmap_key = 0;
25	UINT32 efi_desc_ver = 0;
26	#endif
27
28	bool editor_enabled = true;
29	bool help_hidden = false;
30	bool secure_boot_active = false;
31
32	uint64_t usec_at_bootloader_entry;
33
34	#if defined (UEFI)
35	bool is_efi_serial_present(void) {
36	EFI_STATUS status;
37	EFI_SERIAL_IO_PROTOCOL *serial_io = NULL;
38	EFI_GUID serial_io_guid = EFI_SERIAL_IO_PROTOCOL_GUID;
39
40	status = gBS->LocateProtocol(&serial_io_guid, NULL, (void **)&serial_io);
41	if (status) {
42	return false;
43	}
44
45	if (serial_io == NULL) {
46	return false;
47	}
48
49	UINT32 control;
50	status = serial_io->GetControl(serial_io, &control);
51	if (status) {
52	return false;
53	}
54
55	return true;
56	}
57	#endif
58
59	bool parse_resolution(size_t width, size_t height, size_t bpp, const char buf) {
60	size_t res[3] = {0};
61
62	const char *first = buf;
63	for (size_t i = 0; i < 3; i++) {
64	const char *last;
65	size_t x = strtoui(first, &last, 10);
66	if (first == last)
67	break;
68	res[i] = x;
69	if (*last == 0)
70	break;
71	first = last + 1;
72	}
73
74	if (res[0] == 0 \|\| res[1] == 0)
75	return false;
76
77	if (res[2] == 0)
78	res[2] = 32;
79
80	width = res[0], height = res[1];
81	if (bpp != NULL)
82	*bpp = res[2];
83
84	return true;
85	}
86
87	// This integer sqrt implementation has been adapted from:
88	// https://stackoverflow.com/questions/1100090/looking-for-an-efficient-integer-square-root-algorithm-for-arm-thumb2
89	uint64_t sqrt(uint64_t a_nInput) {
90	uint64_t op = a_nInput;
91	uint64_t res = 0;
92	uint64_t one = (uint64_t)1 << 62;
93
94	// "one" starts at the highest power of four <= than the argument.
95	while (one > op) {
96	one >>= 2;
97	}
98
99	while (one != 0) {
100	if (op >= res + one) {
101	op = op - (res + one);
102	res = res + 2 * one;
103	}
104	res >>= 1;
105	one >>= 2;
106	}
107
108	return res;
109	}
110
111	size_t get_trailing_zeros(uint64_t val) {
112	for (size_t i = 0; i < 64; i++) {
113	if ((val & 1) != 0) {
114	return i;
115	}
116	val >>= 1;
117	}
118	return 64;
119	}
120
121	void get_device_tree_blob(const char config, size_t extra_size,
122	bool measure) {
123	int ret;
124
125	size_t size = 0;
126	void *dtb = NULL;
127
128	{
129	char *dtb_path = NULL;
130	bool soft_panic;
131	if (config != NULL) {
132	dtb_path = config_get_value(config, 0, "dtb_path");
133	soft_panic = true;
134	}
135	if (dtb_path == NULL) {
136	dtb_path = config_get_value(NULL, 0, "global_dtb");
137	soft_panic = false;
138	}
139	if (dtb_path != NULL) {
140	struct file_handle *dtb_file;
141	if ((dtb_file = uri_open(dtb_path, MEMMAP_BOOTLOADER_RECLAIMABLE, false
142	#if defined (__i386__)
143	, NULL, NULL
144	#endif
145	)) == NULL)
146	panic(soft_panic, "dtb: Failed to open device tree blob with path `%#`. Is the path correct?", dtb_path);
147
148	dtb = dtb_file->fd;
149	size = dtb_file->size;
150	fclose(dtb_file);
151
152	ret = fdt_check_full(dtb, size);
153	if (ret != 0) {
154	panic(soft_panic, "dtb: Invalid device tree blob at `%#`: '%s'", dtb_path, fdt_strerror(ret));
155	}
156
157	#if defined (UEFI)
158	if (measure) {
159	tpm_measure_path(TPM_PCR_BOOT_AUTH, TPM_EV_IPL, "dtb_path: ", dtb_path);
160	tpm_measure(TPM_PCR_LOADED_IMAGES, TPM_EV_IPL,
161	dtb, size, "dtb_path: ", dtb_path);
162	}
163	#endif
164
165	printv("dtb: loaded dtb at %p from file `%#`\n", dtb, dtb_path);
166	}
167	}
168
169	#if defined (UEFI)
170	if (!dtb) {
171	EFI_GUID dtb_guid = EFI_DTB_TABLE_GUID;
172	for (size_t i = 0; i < gST->NumberOfTableEntries; i++) {
173	EFI_CONFIGURATION_TABLE *cur_table = &gST->ConfigurationTable[i];
174	if (memcmp(&cur_table->VendorGuid, &dtb_guid, sizeof(EFI_GUID)))
175	continue;
176	size = fdt_totalsize(cur_table->VendorTable);
177	if (measure) {
178	tpm_measure(TPM_PCR_LOADED_IMAGES, TPM_EV_IPL,
179	cur_table->VendorTable, size, "efi_dtb", NULL);
180	}
181	dtb = ext_mem_alloc(size);
182	ret = fdt_open_into(cur_table->VendorTable, dtb, size);
183	if (ret < 0) {
184	panic(true, "dtb: failed to resize new DTB");
185	}
186	printv("dtb: found dtb at %p via EFI\n", cur_table->VendorTable);
187	break;
188	}
189	}
190	#else
191	(void)measure;
192	#endif
193
194	if (extra_size == 0) {
195	return dtb;
196	}
197
198	if (dtb) {
199	printv("dtb: dtb has size %X\n", (uint64_t)size);
200
201	size_t new_size = CHECKED_ADD(size, extra_size,
202	panic(true, "dtb: size overflow"));
203	void *new_tab = ext_mem_alloc(new_size);
204
205	ret = fdt_open_into(dtb, new_tab, new_size);
206	if (ret < 0) {
207	panic(true, "dtb: failed to resize new DTB");
208	}
209
210	pmm_free(dtb, size);
211	return new_tab;
212	}
213
214	dtb = ext_mem_alloc(extra_size);
215
216	ret = fdt_create_empty_tree(dtb, extra_size);
217	if (ret < 0) {
218	panic(true, "dtb: failed to create a device tree blob: '%s'", fdt_strerror(ret));
219	}
220
221	ret = fdt_setprop_u32(dtb, 0, "#address-cells", 2);
222	if (ret < 0) {
223	panic(true, "dtb: failed to set #address-cells: '%s'", fdt_strerror(ret));
224	}
225
226	ret = fdt_setprop_u32(dtb, 0, "#size-cells", 1);
227	if (ret < 0) {
228	panic(true, "dtb: failed to set #size-cells: '%s'", fdt_strerror(ret));
229	}
230
231	return dtb;
232	}
233
234	#if defined (UEFI)
235
236	#if defined (__riscv)
237
238	RISCV_EFI_BOOT_PROTOCOL *get_riscv_boot_protocol(void) {
239	EFI_GUID boot_proto_guid = RISCV_EFI_BOOT_PROTOCOL_GUID;
240	RISCV_EFI_BOOT_PROTOCOL *proto;
241
242	// LocateProtocol() is available from EFI version 1.1
243	if (gBS->Hdr.Revision >= ((1 << 16) \| 10)) {
244	if (gBS->LocateProtocol(&boot_proto_guid, NULL, (void **)&proto) == EFI_SUCCESS) {
245	return proto;
246	}
247	}
248
249	UINTN bufsz = 0;
250	if (gBS->LocateHandle(ByProtocol, &boot_proto_guid, NULL, &bufsz, NULL) != EFI_BUFFER_TOO_SMALL)
251	return NULL;
252
253	UINTN handles_alloc = bufsz;
254	EFI_HANDLE *handles_buf = ext_mem_alloc(handles_alloc);
255	if (handles_buf == NULL)
256	return NULL;
257
258	if (bufsz < sizeof(EFI_HANDLE))
259	goto error;
260
261	if (gBS->LocateHandle(ByProtocol, &boot_proto_guid, NULL, &bufsz, handles_buf) != EFI_SUCCESS)
262	goto error;
263
264	if (gBS->HandleProtocol(handles_buf[0], &boot_proto_guid, (void **)&proto) != EFI_SUCCESS)
265	goto error;
266
267	pmm_free(handles_buf, handles_alloc);
268	return proto;
269
270	error:
271	pmm_free(handles_buf, handles_alloc);
272	return NULL;
273	}
274
275	#endif
276
277	no_unwind bool efi_boot_services_exited = false;
278
279	bool efi_exit_boot_services(void) {
280	EFI_STATUS status;
281
282	// Pull entropy from EFI_RNG_PROTOCOL while it's still callable and
283	// publish it for the kernel to mix into its early RNG state.
284	rng_seed_install();
285
286	// Free the buffer init_memmap left us; the loop below manages
287	// allocation lifetime itself.
288	status = gBS->FreePool(efi_mmap);
289	if (status) {
290	goto fail;
291	}
292	efi_mmap = NULL;
293
294	EFI_MEMORY_DESCRIPTOR *efi_copy = NULL;
295	UINTN efi_mmap_alloc = 0;
296	UINTN efi_copy_alloc = 0;
297
298	bli_on_boot();
299
300	for (size_t retries = 0; ; retries++) {
301	if (retries == 128) {
302	goto fail;
303	}
304
305	efi_mmap_size = efi_mmap_alloc;
306	status = gBS->GetMemoryMap(&efi_mmap_size, efi_mmap, &efi_mmap_key,
307	&efi_desc_size, &efi_desc_ver);
308	if (status == EFI_BUFFER_TOO_SMALL) {
309	// Map grew (or first iteration). Free both buffers and
310	// reallocate, with slack for the descriptors AllocatePool
311	// itself may add.
312	if (efi_mmap != NULL) {
313	gBS->FreePool(efi_mmap);
314	efi_mmap = NULL;
315	}
316	if (efi_copy != NULL) {
317	gBS->FreePool(efi_copy);
318	efi_copy = NULL;
319	}
320	efi_mmap_alloc = efi_mmap_size + 4096;
321	status = gBS->AllocatePool(EfiLoaderData, efi_mmap_alloc,
322	(void **)&efi_mmap);
323	if (status) {
324	goto fail;
325	}
326	efi_copy_alloc = CHECKED_MUL(efi_mmap_alloc, (UINTN)2, goto fail);
327	status = gBS->AllocatePool(EfiLoaderData, efi_copy_alloc,
328	(void **)&efi_copy);
329	if (status) {
330	goto fail;
331	}
332	continue;
333	}
334	if (status) {
335	goto fail;
336	}
337
338	// Be gone, UEFI!
339	status = gBS->ExitBootServices(efi_image_handle, efi_mmap_key);
340	if (status == EFI_SUCCESS) {
341	break;
342	}
343	// Map key invalidated by an allocation - retry.
344	}
345
346	const size_t EFI_COPY_MAX_ENTRIES = efi_copy_alloc / efi_desc_size;
347
348	#if defined(__x86_64__) \|\| defined(__i386__)
349	asm volatile ("cli" ::: "memory");
350	#elif defined (__aarch64__)
351	asm volatile ("msr daifset, #15" ::: "memory");
352	#elif defined (__riscv)
353	asm volatile ("csrci sstatus, 0x2" ::: "memory");
354	#elif defined (__loongarch64)
355	asm volatile ("csrxchg $r0, %0, 0x0" :: "r" (0x4) : "memory");
356	#else
357	#error Unknown architecture
358	#endif
359
360	// Go through new EFI memmap and free up bootloader entries
361	size_t entry_count = efi_mmap_size / efi_desc_size;
362
363	size_t efi_copy_i = 0;
364
365	for (size_t i = 0; i < entry_count; i++) {
366	EFI_MEMORY_DESCRIPTOR orig_entry = (void )efi_mmap + i * efi_desc_size;
367	EFI_MEMORY_DESCRIPTOR new_entry = (void )efi_copy + efi_copy_i * efi_desc_size;
368
369	if (orig_entry->NumberOfPages == 0) {
370	continue;
371	}
372
373	memcpy(new_entry, orig_entry, efi_desc_size);
374
375	uint64_t base = orig_entry->PhysicalStart;
376	uint64_t length = orig_entry->NumberOfPages * 4096;
377	uint64_t top = base + length;
378
379	// Find for a match in the untouched memory map
380	for (size_t j = 0; j < untouched_memmap_entries; j++) {
381	if (untouched_memmap[j].type != MEMMAP_USABLE)
382	continue;
383
384	if (top > untouched_memmap[j].base && top <= untouched_memmap[j].base + untouched_memmap[j].length) {
385	if (untouched_memmap[j].base < base) {
386	new_entry->NumberOfPages = (base - untouched_memmap[j].base) / 4096;
387
388	efi_copy_i++;
389	if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
390	panic(false, "efi: New memory map exhausted");
391	}
392	new_entry = (void )efi_copy + efi_copy_i efi_desc_size;
393	memcpy(new_entry, orig_entry, efi_desc_size);
394
395	new_entry->NumberOfPages -= (base - untouched_memmap[j].base) / 4096;
396	new_entry->PhysicalStart = base;
397	new_entry->VirtualStart = 0;
398
399	length = new_entry->NumberOfPages * 4096;
400	top = base + length;
401	}
402
403	if (untouched_memmap[j].base > base) {
404	new_entry->NumberOfPages = (untouched_memmap[j].base - base) / 4096;
405
406	efi_copy_i++;
407	if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
408	panic(false, "efi: New memory map exhausted");
409	}
410	new_entry = (void )efi_copy + efi_copy_i efi_desc_size;
411	memcpy(new_entry, orig_entry, efi_desc_size);
412
413	new_entry->NumberOfPages -= (untouched_memmap[j].base - base) / 4096;
414	new_entry->PhysicalStart = untouched_memmap[j].base;
415	new_entry->VirtualStart = 0;
416
417	base = new_entry->PhysicalStart;
418	length = new_entry->NumberOfPages * 4096;
419	top = base + length;
420	}
421
422	if (length < untouched_memmap[j].length) {
423	panic(false, "efi: Memory map corruption");
424	}
425
426	new_entry->Type = EfiConventionalMemory;
427
428	if (length == untouched_memmap[j].length) {
429	// It's a perfect match!
430	break;
431	}
432
433	new_entry->NumberOfPages = untouched_memmap[j].length / 4096;
434
435	efi_copy_i++;
436	if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
437	panic(false, "efi: New memory map exhausted");
438	}
439	new_entry = (void )efi_copy + efi_copy_i efi_desc_size;
440	memcpy(new_entry, orig_entry, efi_desc_size);
441
442	new_entry->NumberOfPages = (length - untouched_memmap[j].length) / 4096;
443	new_entry->PhysicalStart = base + untouched_memmap[j].length;
444	new_entry->VirtualStart = 0;
445
446	break;
447	}
448	}
449
450	efi_copy_i++;
451	if (efi_copy_i == EFI_COPY_MAX_ENTRIES) {
452	panic(false, "efi: New memory map exhausted");
453	}
454	}
455
456	efi_mmap = efi_copy;
457	efi_mmap_size = efi_copy_i * efi_desc_size;
458
459	efi_boot_services_exited = true;
460
461	printv("efi: Exited boot services.\n");
462
463	return true;
464
465	fail:
466	panic(false, "efi: Failed to exit boot services");
467	}
468
469	#endif