arm64: demand_paging: add support for on-demand mappings

This makes ARM64 compatible with K_MEM_MAP_UNPAGED.

(cherry picked from commit be279d8)

Original-Signed-off-by: Nicolas Pitre <npitre@baylibre.com>
GitOrigin-RevId: be279d8
Cr-Build-Id: 8736917283475341809
Cr-Build-Url: https://cr-buildbucket.appspot.com/build/8736917283475341809
Copybot-Job-Name: zephyr-main-copybot-downstream
Change-Id: Ia5f91ce197b6395ec6f88cc7102dc1e62e0dd101
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/zephyr/+/5856312
Commit-Queue: Keith Short <keithshort@chromium.org>
Tested-by: ChromeOS Prod (Robot) <chromeos-ci-prod@chromeos-bot.iam.gserviceaccount.com>
Reviewed-by: Keith Short <keithshort@chromium.org>

arch/arm64/core/mmu.c

@@ -243,8 +243,15 @@ static void set_pte_table_desc(uint64_t *pte, uint64_t *table, unsigned int leve
 
 static void set_pte_block_desc(uint64_t *pte, uint64_t desc, unsigned int level)
 {
- if (desc) {
- desc |= (level == XLAT_LAST_LEVEL) ? PTE_PAGE_DESC : PTE_BLOCK_DESC;
+ if (level != XLAT_LAST_LEVEL) {
+ desc |= PTE_BLOCK_DESC;
+ } else if (!IS_ENABLED(CONFIG_DEMAND_PAGING) || (desc & PTE_BLOCK_DESC_AF) != 0) {
+ desc |= PTE_PAGE_DESC;
+ } else {
+ /*
+ * Demand paging configured and AF unset: leave the descriptor
+ * type to "invalid" as in arch_mem_page_out().
+ */
  }
  *pte = desc;
  debug_show_pte(pte, level);
@@ -677,6 +684,11 @@ static uint64_t get_region_desc(uint32_t attrs)
 
  /* the access flag */
  desc |= PTE_BLOCK_DESC_AF;
+ if (IS_ENABLED(CONFIG_DEMAND_PAGING) && (attrs & MT_PAGED_OUT) != 0) {
+ /* set it up for demand paging like arch_mem_page_out() */
+ desc &= ~PTE_BLOCK_DESC_AF;
+ desc |= PTE_BLOCK_DESC_AP_RO;
+ }
 
  /* memory attribute index field */
  mem_type = MT_TYPE(attrs);
@@ -1090,6 +1102,10 @@ static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flag
  entry_flags |= MT_RW_AP_ELx;
  }
 
+ if (IS_ENABLED(CONFIG_DEMAND_PAGING) && (flags & K_MEM_MAP_UNPAGED) != 0) {
+ entry_flags |= MT_PAGED_OUT;
+ }
+
  return add_map(ptables, "generic", phys, (uintptr_t)virt, size, entry_flags);
 }
 

include/zephyr/arch/arm64/arm_mmu.h

@@ -45,6 +45,7 @@
  * attrs[7] : Mirror RO/RW permissions to EL0
  * attrs[8] : Overwrite existing mapping if any
  * attrs[9] : non-Global mapping (nG)
+ * attrs[10]: Paged-out mapping
  *
  */
 #define MT_PERM_SHIFT 3U
@@ -54,6 +55,7 @@
 #define MT_RW_AP_SHIFT 7U
 #define MT_NO_OVERWRITE_SHIFT 8U
 #define MT_NON_GLOBAL_SHIFT 9U
+#define MT_PAGED_OUT_SHIFT 10U
 
 #define MT_RO (0U << MT_PERM_SHIFT)
 #define MT_RW (1U << MT_PERM_SHIFT)
@@ -75,6 +77,8 @@
 #define MT_G (0U << MT_NON_GLOBAL_SHIFT)
 #define MT_NG (1U << MT_NON_GLOBAL_SHIFT)
 
+#define MT_PAGED_OUT (1U << MT_PAGED_OUT_SHIFT)
+
 #define MT_P_RW_U_RW (MT_RW | MT_RW_AP_ELx | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
 #define MT_P_RW_U_NA (MT_RW | MT_RW_AP_EL_HIGHER | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
 #define MT_P_RO_U_RO (MT_RO | MT_RW_AP_ELx | MT_P_EXECUTE_NEVER | MT_U_EXECUTE_NEVER)
@@ -95,6 +99,13 @@
 #define ARCH_DATA_PAGE_DIRTY BIT(2)
 #define ARCH_DATA_PAGE_NOT_MAPPED BIT(3)
 
+/*
+ * Special unpaged "location" tags (highest possible descriptor physical
+ * address values unlikely to conflict with backing store locations)
+ */
+#define ARCH_UNPAGED_ANON_ZERO 0x0000fffffffff000
+#define ARCH_UNPAGED_ANON_UNINIT 0x0000ffffffffe000
+
 #ifndef _ASMLANGUAGE
 
 /* Region definition data structure */