arm64: demand_paging: add support for on-demand mappings

This makes ARM64 compatible with K_MEM_MAP_UNPAGED.

(cherry picked from commit be279d8)

Original-Signed-off-by: Nicolas Pitre <npitre@baylibre.com>
GitOrigin-RevId: be279d8
Cr-Build-Id: 8736917283475341809
Cr-Build-Url: https://cr-buildbucket.appspot.com/build/8736917283475341809
Copybot-Job-Name: zephyr-main-copybot-downstream
Change-Id: Ia5f91ce197b6395ec6f88cc7102dc1e62e0dd101
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/zephyr/+/5856312
Commit-Queue: Keith Short <keithshort@chromium.org>
Tested-by: ChromeOS Prod (Robot) <chromeos-ci-prod@chromeos-bot.iam.gserviceaccount.com>
Reviewed-by: Keith Short <keithshort@chromium.org>

arch/arm64/core/mmu.c

@@ -243,8 +243,15 @@ static void set_pte_table_desc(uint64_t *pte, uint64_t *table, unsigned int leve
 
 static void set_pte_block_desc(uint64_t *pte, uint64_t desc, unsigned int level)
 {
-	if (desc) {
-		desc |= (level == XLAT_LAST_LEVEL) ? PTE_PAGE_DESC : PTE_BLOCK_DESC;
+	if (level != XLAT_LAST_LEVEL) {
+		desc |= PTE_BLOCK_DESC;
+	} else if (!IS_ENABLED(CONFIG_DEMAND_PAGING) || (desc & PTE_BLOCK_DESC_AF) != 0) {
+		desc |= PTE_PAGE_DESC;
+	} else {
+		/*
+		 * Demand paging configured and AF unset: leave the descriptor
+		 * type to "invalid" as in arch_mem_page_out().
+		 */
 	}
 	*pte = desc;
 	debug_show_pte(pte, level);
@@ -677,6 +684,11 @@ static uint64_t get_region_desc(uint32_t attrs)
 
 	/* the access flag */
 	desc |= PTE_BLOCK_DESC_AF;
+	if (IS_ENABLED(CONFIG_DEMAND_PAGING) && (attrs & MT_PAGED_OUT) != 0) {
+		/* set it up for demand paging like arch_mem_page_out() */
+		desc &= ~PTE_BLOCK_DESC_AF;
+		desc |= PTE_BLOCK_DESC_AP_RO;
+	}
 
 	/* memory attribute index field */
 	mem_type = MT_TYPE(attrs);
@@ -1090,6 +1102,10 @@ static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flag
 		entry_flags |= MT_RW_AP_ELx;
 	}
 
+	if (IS_ENABLED(CONFIG_DEMAND_PAGING) && (flags & K_MEM_MAP_UNPAGED) != 0) {
+		entry_flags |= MT_PAGED_OUT;
+	}
+
 	return add_map(ptables, "generic", phys, (uintptr_t)virt, size, entry_flags);
 }
 

include/zephyr/arch/arm64/arm_mmu.h

@@ -45,6 +45,7 @@
  * attrs[7] : Mirror RO/RW permissions to EL0
  * attrs[8] : Overwrite existing mapping if any
  * attrs[9] : non-Global mapping (nG)
+ * attrs[10]: Paged-out mapping
  *
  */
 #define MT_PERM_SHIFT		3U
@@ -54,6 +55,7 @@
 #define MT_RW_AP_SHIFT		7U
 #define MT_NO_OVERWRITE_SHIFT	8U
 #define MT_NON_GLOBAL_SHIFT	9U
+#define MT_PAGED_OUT_SHIFT	10U
 
 #define MT_RO			(0U << MT_PERM_SHIFT)
 #define MT_RW			(1U << MT_PERM_SHIFT)
@@ -75,6 +77,8 @@
 #define MT_G			(0U << MT_NON_GLOBAL_SHIFT)
 #define MT_NG			(1U << MT_NON_GLOBAL_SHIFT)
 
+#define MT_PAGED_OUT		(1U << MT_PAGED_OUT_SHIFT)
+
 #define MT_P_RW_U_RW		(MT_RW | MT_RW_AP_ELx | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
 #define MT_P_RW_U_NA		(MT_RW | MT_RW_AP_EL_HIGHER  | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
 #define MT_P_RO_U_RO		(MT_RO | MT_RW_AP_ELx | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
@@ -95,6 +99,13 @@
 #define ARCH_DATA_PAGE_DIRTY		BIT(2)
 #define ARCH_DATA_PAGE_NOT_MAPPED	BIT(3)
 
+/*
+ * Special unpaged "location" tags (highest possible descriptor physical
+ * address values unlikely to conflict with backing store locations)
+ */
+#define ARCH_UNPAGED_ANON_ZERO		0x0000fffffffff000
+#define ARCH_UNPAGED_ANON_UNINIT	0x0000ffffffffe000
+
 #ifndef _ASMLANGUAGE
 
 /* Region definition data structure */