|
|
@@ -34,6 +34,7 @@
|
|
|
#include <linux/slab.h>
|
|
|
#include <linux/ucs2_string.h>
|
|
|
#include <linux/mem_encrypt.h>
|
|
|
+#include <linux/sched/task.h>
|
|
|
|
|
|
#include <asm/setup.h>
|
|
|
#include <asm/page.h>
|
|
|
@@ -82,9 +83,8 @@ pgd_t * __init efi_call_phys_prolog(void)
|
|
|
int n_pgds, i, j;
|
|
|
|
|
|
if (!efi_enabled(EFI_OLD_MEMMAP)) {
|
|
|
- save_pgd = (pgd_t *)__read_cr3();
|
|
|
- write_cr3((unsigned long)efi_scratch.efi_pgt);
|
|
|
- goto out;
|
|
|
+ efi_switch_mm(&efi_mm);
|
|
|
+ return NULL;
|
|
|
}
|
|
|
|
|
|
early_code_mapping_set_exec(1);
|
|
|
@@ -156,8 +156,7 @@ void __init efi_call_phys_epilog(pgd_t *save_pgd)
|
|
|
pud_t *pud;
|
|
|
|
|
|
if (!efi_enabled(EFI_OLD_MEMMAP)) {
|
|
|
- write_cr3((unsigned long)save_pgd);
|
|
|
- __flush_tlb_all();
|
|
|
+ efi_switch_mm(efi_scratch.prev_mm);
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
@@ -191,8 +190,7 @@ void __init efi_call_phys_epilog(pgd_t *save_pgd)
|
|
|
early_code_mapping_set_exec(0);
|
|
|
}
|
|
|
|
|
|
-pgd_t *efi_pgd;
|
|
|
-EXPORT_SYMBOL_GPL(efi_pgd);
|
|
|
+EXPORT_SYMBOL_GPL(efi_mm);
|
|
|
|
|
|
/*
|
|
|
* We need our own copy of the higher levels of the page tables
|
|
|
@@ -205,7 +203,7 @@ EXPORT_SYMBOL_GPL(efi_pgd);
|
|
|
*/
|
|
|
int __init efi_alloc_page_tables(void)
|
|
|
{
|
|
|
- pgd_t *pgd;
|
|
|
+ pgd_t *pgd, *efi_pgd;
|
|
|
p4d_t *p4d;
|
|
|
pud_t *pud;
|
|
|
gfp_t gfp_mask;
|
|
|
@@ -233,6 +231,10 @@ int __init efi_alloc_page_tables(void)
|
|
|
return -ENOMEM;
|
|
|
}
|
|
|
|
|
|
+ efi_mm.pgd = efi_pgd;
|
|
|
+ mm_init_cpumask(&efi_mm);
|
|
|
+ init_new_context(NULL, &efi_mm);
|
|
|
+
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
@@ -245,6 +247,7 @@ void efi_sync_low_kernel_mappings(void)
|
|
|
pgd_t *pgd_k, *pgd_efi;
|
|
|
p4d_t *p4d_k, *p4d_efi;
|
|
|
pud_t *pud_k, *pud_efi;
|
|
|
+ pgd_t *efi_pgd = efi_mm.pgd;
|
|
|
|
|
|
if (efi_enabled(EFI_OLD_MEMMAP))
|
|
|
return;
|
|
|
@@ -338,19 +341,11 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
|
|
|
unsigned long pfn, text, pf;
|
|
|
struct page *page;
|
|
|
unsigned npages;
|
|
|
- pgd_t *pgd;
|
|
|
+ pgd_t *pgd = efi_mm.pgd;
|
|
|
|
|
|
if (efi_enabled(EFI_OLD_MEMMAP))
|
|
|
return 0;
|
|
|
|
|
|
- /*
|
|
|
- * Since the PGD is encrypted, set the encryption mask so that when
|
|
|
- * this value is loaded into cr3 the PGD will be decrypted during
|
|
|
- * the pagetable walk.
|
|
|
- */
|
|
|
- efi_scratch.efi_pgt = (pgd_t *)__sme_pa(efi_pgd);
|
|
|
- pgd = efi_pgd;
|
|
|
-
|
|
|
/*
|
|
|
* It can happen that the physical address of new_memmap lands in memory
|
|
|
* which is not mapped in the EFI page table. Therefore we need to go
|
|
|
@@ -364,8 +359,6 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
|
|
|
return 1;
|
|
|
}
|
|
|
|
|
|
- efi_scratch.use_pgd = true;
|
|
|
-
|
|
|
/*
|
|
|
* Certain firmware versions are way too sentimential and still believe
|
|
|
* they are exclusive and unquestionable owners of the first physical page,
|
|
|
@@ -419,7 +412,7 @@ static void __init __map_region(efi_memory_desc_t *md, u64 va)
|
|
|
{
|
|
|
unsigned long flags = _PAGE_RW;
|
|
|
unsigned long pfn;
|
|
|
- pgd_t *pgd = efi_pgd;
|
|
|
+ pgd_t *pgd = efi_mm.pgd;
|
|
|
|
|
|
if (!(md->attribute & EFI_MEMORY_WB))
|
|
|
flags |= _PAGE_PCD;
|
|
|
@@ -523,7 +516,7 @@ void __init parse_efi_setup(u64 phys_addr, u32 data_len)
|
|
|
static int __init efi_update_mappings(efi_memory_desc_t *md, unsigned long pf)
|
|
|
{
|
|
|
unsigned long pfn;
|
|
|
- pgd_t *pgd = efi_pgd;
|
|
|
+ pgd_t *pgd = efi_mm.pgd;
|
|
|
int err1, err2;
|
|
|
|
|
|
/* Update the 1:1 mapping */
|
|
|
@@ -620,10 +613,26 @@ void __init efi_dump_pagetable(void)
|
|
|
if (efi_enabled(EFI_OLD_MEMMAP))
|
|
|
ptdump_walk_pgd_level(NULL, swapper_pg_dir);
|
|
|
else
|
|
|
- ptdump_walk_pgd_level(NULL, efi_pgd);
|
|
|
+ ptdump_walk_pgd_level(NULL, efi_mm.pgd);
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * Makes the calling thread switch to/from efi_mm context. Can be used
|
|
|
+ * for SetVirtualAddressMap() i.e. current->active_mm == init_mm as well
|
|
|
+ * as during efi runtime calls i.e current->active_mm == current_mm.
|
|
|
+ * We are not mm_dropping()/mm_grabbing() any mm, because we are not
|
|
|
+ * losing/creating any references.
|
|
|
+ */
|
|
|
+void efi_switch_mm(struct mm_struct *mm)
|
|
|
+{
|
|
|
+ task_lock(current);
|
|
|
+ efi_scratch.prev_mm = current->active_mm;
|
|
|
+ current->active_mm = mm;
|
|
|
+ switch_mm(efi_scratch.prev_mm, mm, NULL);
|
|
|
+ task_unlock(current);
|
|
|
+}
|
|
|
+
|
|
|
#ifdef CONFIG_EFI_MIXED
|
|
|
extern efi_status_t efi64_thunk(u32, ...);
|
|
|
|
|
|
@@ -677,16 +686,13 @@ efi_status_t efi_thunk_set_virtual_address_map(
|
|
|
efi_sync_low_kernel_mappings();
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
- efi_scratch.prev_cr3 = __read_cr3();
|
|
|
- write_cr3((unsigned long)efi_scratch.efi_pgt);
|
|
|
- __flush_tlb_all();
|
|
|
+ efi_switch_mm(&efi_mm);
|
|
|
|
|
|
func = (u32)(unsigned long)phys_set_virtual_address_map;
|
|
|
status = efi64_thunk(func, memory_map_size, descriptor_size,
|
|
|
descriptor_version, virtual_map);
|
|
|
|
|
|
- write_cr3(efi_scratch.prev_cr3);
|
|
|
- __flush_tlb_all();
|
|
|
+ efi_switch_mm(efi_scratch.prev_mm);
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
return status;
|