Skip to content

Commit

Permalink
arm64: head: cover entire kernel image in initial ID map
Browse files Browse the repository at this point in the history
As a first step towards avoiding the need to create, tear down and
recreate the kernel virtual mapping with MMU and caches disabled, start
by expanding the ID map so it covers the page tables as well as all
executable code. This will allow us to populate the page tables with the
MMU and caches on, and call KASLR init code before setting up the
virtual mapping.

Since this ID map is only needed at boot, create it as a temporary set
of page tables, and populate the permanent ID map after enabling the MMU
and caches. While at it, switch to read-only attributes for the where
possible, as writable permissions are only needed for the initial kernel
page tables. Note that on 4k granule configurations, the permanent ID
map will now be reduced to a single page rather than a 2M block mapping.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220624150651.1358849-13-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
  • Loading branch information
ardbiesheuvel authored and willdeacon committed Jun 24, 2022

Verified

This commit was signed with the committer’s verified signature.
WarstekHUN Levente Balogh
1 parent b013c1e commit c3cee92
Showing 5 changed files with 76 additions and 21 deletions.
16 changes: 11 additions & 5 deletions arch/arm64/include/asm/kernel-pgtable.h
Original file line number Diff line number Diff line change
@@ -35,10 +35,8 @@
*/
#if ARM64_KERNEL_USES_PMD_MAPS
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1)
#else
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
#endif


@@ -87,7 +85,13 @@
+ EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \
+ EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */
#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end))
#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)

/* the initial ID map may need two extra pages if it needs to be extended */
#if VA_BITS < 48
#define INIT_IDMAP_DIR_SIZE (INIT_DIR_SIZE + (2 * PAGE_SIZE))
#else
#define INIT_IDMAP_DIR_SIZE INIT_DIR_SIZE
#endif

/* Initial memory map size */
#if ARM64_KERNEL_USES_PMD_MAPS
@@ -107,9 +111,11 @@
#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)

#if ARM64_KERNEL_USES_PMD_MAPS
#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
#define SWAPPER_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PMD_SECT_RDONLY)
#else
#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
#define SWAPPER_RW_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY)
#endif

/*
31 changes: 21 additions & 10 deletions arch/arm64/kernel/head.S
Original file line number Diff line number Diff line change
@@ -87,6 +87,7 @@
* x28 clear_page_tables() callee preserved temp register
* x19/x20 __primary_switch() callee preserved temp registers
* x24 __primary_switch() .. relocate_kernel() current RELR displacement
* x28 create_idmap() callee preserved temp register
*/
SYM_CODE_START(primary_entry)
bl preserve_boot_args
@@ -298,9 +299,7 @@ SYM_FUNC_START_LOCAL(remap_region)
SYM_FUNC_END(remap_region)

SYM_FUNC_START_LOCAL(create_idmap)
adrp x0, idmap_pg_dir
adrp x3, __idmap_text_start // __pa(__idmap_text_start)

mov x28, lr
/*
* The ID map carries a 1:1 mapping of the physical address range
* covered by the loaded image, which could be anywhere in DRAM. This
@@ -347,21 +346,33 @@ SYM_FUNC_START_LOCAL(create_idmap)
* translation level, but the top-level table has more entries.
*/
#endif
adr_l x6, __idmap_text_end // __pa(__idmap_text_end)
mov x7, SWAPPER_MM_MMUFLAGS
adrp x0, init_idmap_pg_dir
adrp x3, _text
adrp x6, _end
mov x7, SWAPPER_RX_MMUFLAGS

map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT

/* Remap the kernel page tables r/w in the ID map */
adrp x1, _text
adrp x2, init_pg_dir
adrp x3, init_pg_end
bic x4, x2, #SWAPPER_BLOCK_SIZE - 1
mov x5, SWAPPER_RW_MMUFLAGS
mov x6, #SWAPPER_BLOCK_SHIFT
bl remap_region

/*
* Since the page tables have been populated with non-cacheable
* accesses (MMU disabled), invalidate those tables again to
* remove any speculatively loaded cache lines.
*/
dmb sy

adrp x0, idmap_pg_dir
adrp x1, idmap_pg_end
b dcache_inval_poc // tail call
adrp x0, init_idmap_pg_dir
adrp x1, init_idmap_pg_end
bl dcache_inval_poc
ret x28
SYM_FUNC_END(create_idmap)

SYM_FUNC_START_LOCAL(create_kernel_mapping)
@@ -372,7 +383,7 @@ SYM_FUNC_START_LOCAL(create_kernel_mapping)
adrp x3, _text // runtime __pa(_text)
sub x6, x6, x3 // _end - _text
add x6, x6, x5 // runtime __va(_end)
mov x7, SWAPPER_MM_MMUFLAGS
mov x7, SWAPPER_RW_MMUFLAGS

map_memory x0, x1, x5, x6, x7, x3, (VA_BITS - PGDIR_SHIFT), x10, x11, x12, x13, x14

@@ -853,7 +864,7 @@ SYM_FUNC_START_LOCAL(__primary_switch)
#endif

adrp x1, init_pg_dir
adrp x2, idmap_pg_dir
adrp x2, init_idmap_pg_dir
bl __enable_mmu
#ifdef CONFIG_RELOCATABLE
#ifdef CONFIG_RELR
7 changes: 5 additions & 2 deletions arch/arm64/kernel/vmlinux.lds.S
Original file line number Diff line number Diff line change
@@ -198,8 +198,7 @@ SECTIONS
}

idmap_pg_dir = .;
. += IDMAP_DIR_SIZE;
idmap_pg_end = .;
. += PAGE_SIZE;

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
tramp_pg_dir = .;
@@ -235,6 +234,10 @@ SECTIONS
__inittext_end = .;
__initdata_begin = .;

init_idmap_pg_dir = .;
. += INIT_IDMAP_DIR_SIZE;
init_idmap_pg_end = .;

.init.data : {
INIT_DATA
INIT_SETUP(16)
35 changes: 34 additions & 1 deletion arch/arm64/mm/mmu.c
Original file line number Diff line number Diff line change
@@ -766,9 +766,40 @@ static void __init map_kernel(pgd_t *pgdp)
kasan_copy_shadow(pgdp);
}

static void __init create_idmap(void)
{
u64 start = __pa_symbol(__idmap_text_start);
u64 size = __pa_symbol(__idmap_text_end) - start;
pgd_t *pgd = idmap_pg_dir;
u64 pgd_phys;

/* check if we need an additional level of translation */
if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) {
pgd_phys = early_pgtable_alloc(PAGE_SHIFT);
set_pgd(&idmap_pg_dir[start >> VA_BITS],
__pgd(pgd_phys | P4D_TYPE_TABLE));
pgd = __va(pgd_phys);
}
__create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX,
early_pgtable_alloc, 0);

if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
extern u32 __idmap_kpti_flag;
u64 pa = __pa_symbol(&__idmap_kpti_flag);

/*
* The KPTI G-to-nG conversion code needs a read-write mapping
* of its synchronization flag in the ID map.
*/
__create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL,
early_pgtable_alloc, 0);
}
}

void __init paging_init(void)
{
pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir));
extern pgd_t init_idmap_pg_dir[];

idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0));

@@ -777,13 +808,15 @@ void __init paging_init(void)

pgd_clear_fixmap();

cpu_replace_ttbr1(lm_alias(swapper_pg_dir), idmap_pg_dir);
cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir);
init_mm.pgd = swapper_pg_dir;

memblock_phys_free(__pa_symbol(init_pg_dir),
__pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir));

memblock_allow_resize();

create_idmap();
}

/*
8 changes: 5 additions & 3 deletions arch/arm64/mm/proc.S
Original file line number Diff line number Diff line change
@@ -222,8 +222,10 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1)
*
* Called exactly once from stop_machine context by each CPU found during boot.
*/
__idmap_kpti_flag:
.long 1
.pushsection ".data", "aw", %progbits
SYM_DATA(__idmap_kpti_flag, .long 1)
.popsection

SYM_FUNC_START(idmap_kpti_install_ng_mappings)
cpu .req w0
num_cpus .req w1
@@ -245,7 +247,7 @@ SYM_FUNC_START(idmap_kpti_install_ng_mappings)

mrs swapper_ttb, ttbr1_el1
restore_ttbr1 swapper_ttb
adr flag_ptr, __idmap_kpti_flag
adr_l flag_ptr, __idmap_kpti_flag

cbnz cpu, __idmap_kpti_secondary

0 comments on commit c3cee92

Please sign in to comment.