1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/paging/table_ops.hpp"
4
5#include "mos/mm/mm.hpp"
6#include "mos/mm/mmstat.hpp"
7#include "mos/mm/paging/pml_types.hpp"
8#include "mos/mm/paging/pmlx/pml1.hpp"
9#include "mos/mm/paging/pmlx/pml2.hpp"
10#include "mos/mm/paging/pmlx/pml3.hpp"
11#include "mos/mm/paging/pmlx/pml4.hpp"
12#include "mos/mm/paging/pmlx/pml5.hpp"
13#include "mos/mm/paging/table_ops/do_copy.hpp"
14#include "mos/mm/paging/table_ops/do_flag.hpp"
15#include "mos/mm/paging/table_ops/do_map.hpp"
16#include "mos/mm/paging/table_ops/do_mask.hpp"
17#include "mos/mm/paging/table_ops/do_unmap.hpp"
18#include "mos/platform/platform.hpp"
19#include "mos/syslog/printk.hpp"
20
21#include <mos/types.hpp>
22
23void mm_do_map(pgd_t pgd, ptr_t vaddr, pfn_t pfn, size_t n_pages, VMFlags flags, bool do_refcount)
24{
25 struct pagetable_do_map_data data = { .pfn = pfn, .flags = flags, .do_refcount = do_refcount };
26 pml5_traverse(pml5: pgd.max, vaddr: &vaddr, n_pages: &n_pages, callback: pagetable_do_map_callbacks, data: &data);
27}
28
29void mm_do_flag(pgd_t max, ptr_t vaddr, size_t n_pages, VMFlags flags)
30{
31 struct pagetable_do_flag_data data = { .flags = flags };
32 pml5_traverse(pml5: max.max, vaddr: &vaddr, n_pages: &n_pages, callback: pagetable_do_flag_callbacks, data: &data);
33}
34
35void mm_do_unmap(pgd_t max, ptr_t vaddr, size_t n_pages, bool do_unref)
36{
37 pr_dinfo2(vmm, "mm_do_unmap: vaddr=" PTR_FMT ", n_pages=%zu, do_unref=%d", vaddr, n_pages, do_unref);
38 ptr_t vaddr1 = vaddr;
39 size_t n_pages1 = n_pages;
40
41 const ptr_t vaddr2 = vaddr;
42 const size_t n_pages2 = n_pages;
43
44 struct pagetable_do_unmap_data data = { .do_unref = do_unref };
45 pml5_traverse(pml5: max.max, vaddr: &vaddr, n_pages: &n_pages, callback: pagetable_do_unmap_callbacks, data: &data);
46 bool pml5_destroyed = pml5_destroy_range(pml5: max.max, vaddr: &vaddr1, n_pages: &n_pages1);
47 if (pml5_destroyed)
48 pr_warn("mm_do_unmap: pml5 destroyed: vaddr=" PTR_RANGE ", n_pages=%zu", vaddr2, vaddr2 + n_pages2 * MOS_PAGE_SIZE, n_pages2);
49}
50
51void mm_do_mask_flags(pgd_t max, ptr_t vaddr, size_t n_pages, VMFlags mask)
52{
53 struct pagetable_do_mask_data data = { .mask = mask };
54 pml5_traverse(pml5: max.max, vaddr: &vaddr, n_pages: &n_pages, callback: pagetable_do_mask_callbacks, data: &data);
55}
56
57void mm_do_copy(pgd_t src, pgd_t dst, ptr_t vaddr, size_t n_pages)
58{
59 struct pagetable_do_copy_data data = {
60 .dest_pml5 = dst.max,
61 .dest_pml5e = pml5_entry(pml5: dst.max, vaddr),
62 .dest_pml4 = pml5e_get_or_create_pml4(pml5e: data.dest_pml5e),
63 };
64 pml5_traverse(pml5: src.max, vaddr: &vaddr, n_pages: &n_pages, callback: pagetable_do_copy_callbacks, data: &data);
65}
66
67pfn_t mm_do_get_pfn(pgd_t max, ptr_t vaddr)
68{
69 vaddr = ALIGN_DOWN_TO_PAGE(vaddr);
70 pml5e_t *pml5e = pml5_entry(pml5: max.max, vaddr);
71 if (!pml5e_is_present(pml5e))
72 return 0;
73
74 const pml4_t pml4 = pml5e_get_or_create_pml4(pml5e);
75 pml4e_t *pml4e = pml4_entry(pml4, vaddr);
76 if (!pml4e_is_present(pml4e))
77 return 0;
78
79#if MOS_CONFIG(PML4_HUGE_CAPABLE)
80 if (platform_pml4e_is_huge(pml4e))
81 return platform_pml4e_get_huge_pfn(pml4e) + (vaddr & PML4_HUGE_MASK) / MOS_PAGE_SIZE;
82#endif
83
84 const pml3_t pml3 = pml4e_get_or_create_pml3(pml4e);
85 pml3e_t *pml3e = pml3_entry(pml3, vaddr);
86 if (!pml3e_is_present(pml3e))
87 return 0;
88
89#if MOS_CONFIG(PML3_HUGE_CAPABLE)
90 if (platform_pml3e_is_huge(pml3e))
91 return platform_pml3e_get_huge_pfn(pml3e) + (vaddr & PML3_HUGE_MASK) / MOS_PAGE_SIZE;
92#endif
93
94 const pml2_t pml2 = pml3e_get_or_create_pml2(pml3e);
95 pml2e_t *pml2e = pml2_entry(pml2, vaddr);
96 if (!pml2e_is_present(pml2e))
97 return 0;
98
99#if MOS_CONFIG(PML2_HUGE_CAPABLE)
100 if (platform_pml2e_is_huge(pml2: pml2e))
101 return platform_pml2e_get_huge_pfn(pml2: pml2e) + (vaddr & PML2_HUGE_MASK) / MOS_PAGE_SIZE;
102#endif
103
104 const pml1_t pml1 = pml2e_get_or_create_pml1(pml2e);
105 const pml1e_t *pml1e = pml1_entry(pml1, vaddr);
106 if (!pml1e_is_present(pml1e))
107 return 0;
108
109 return pml1e_get_pfn(pml1e);
110}
111
112VMFlags mm_do_get_flags(pgd_t max, ptr_t vaddr)
113{
114 VMFlags flags = VMFlags::all();
115 vaddr = ALIGN_DOWN_TO_PAGE(vaddr);
116 pml5e_t *pml5e = pml5_entry(pml5: max.max, vaddr);
117 if (!pml5e_is_present(pml5e))
118 return VM_NONE;
119
120 const pml4_t pml4 = pml5e_get_or_create_pml4(pml5e);
121 pml4e_t *pml4e = pml4_entry(pml4, vaddr);
122 if (!pml4e_is_present(pml4e))
123 return VM_NONE;
124
125#if MOS_CONFIG(PML4_HUGE_CAPABLE)
126 if (platform_pml4e_is_huge(pml4e))
127 return platform_pml4e_get_flags(pml4e);
128 else
129 flags &= platform_pml4e_get_flags(pml4e);
130#else
131 flags &= platform_pml4e_get_flags(pml4e);
132#endif
133
134 const pml3_t pml3 = pml4e_get_or_create_pml3(pml4e);
135 pml3e_t *pml3e = pml3_entry(pml3, vaddr);
136 if (!pml3e_is_present(pml3e))
137 return VM_NONE;
138
139#if MOS_CONFIG(PML3_HUGE_CAPABLE)
140 if (platform_pml3e_is_huge(pml3e))
141 return platform_pml3e_get_flags(pml3e);
142 else
143 flags &= platform_pml3e_get_flags(pml3e);
144#else
145 flags &= platform_pml3e_get_flags(pml3e);
146#endif
147
148 const pml2_t pml2 = pml3e_get_or_create_pml2(pml3e);
149 pml2e_t *pml2e = pml2_entry(pml2, vaddr);
150 if (!pml2e_is_present(pml2e))
151 return VM_NONE;
152
153#if MOS_CONFIG(PML2_HUGE_CAPABLE)
154 if (platform_pml2e_is_huge(pml2: pml2e))
155 return platform_pml2e_get_flags(pml2e);
156 else
157 flags &= platform_pml2e_get_flags(pml2e);
158#else
159 flags &= platform_pml2e_get_flags(pml2e);
160#endif
161
162 const pml1_t pml1 = pml2e_get_or_create_pml1(pml2e);
163 const pml1e_t *pml1e = pml1_entry(pml1, vaddr);
164 if (!pml1e_is_present(pml1e))
165 return VM_NONE;
166
167 flags &= platform_pml1e_get_flags(pml1e);
168 return flags;
169}
170
171bool mm_do_get_present(pgd_t max, ptr_t vaddr)
172{
173 vaddr = ALIGN_DOWN_TO_PAGE(vaddr);
174 pml5e_t *pml5e = pml5_entry(pml5: max.max, vaddr);
175 if (!pml5e_is_present(pml5e))
176 return false;
177
178 const pml4_t pml4 = pml5e_get_or_create_pml4(pml5e);
179 pml4e_t *pml4e = pml4_entry(pml4, vaddr);
180 if (!pml4e_is_present(pml4e))
181 return false;
182
183 const pml3_t pml3 = pml4e_get_or_create_pml3(pml4e);
184 pml3e_t *pml3e = pml3_entry(pml3, vaddr);
185 if (!pml3e_is_present(pml3e))
186 return false;
187
188 const pml2_t pml2 = pml3e_get_or_create_pml2(pml3e);
189 pml2e_t *pml2e = pml2_entry(pml2, vaddr);
190 if (!pml2e_is_present(pml2e))
191 return false;
192
193 const pml1_t pml1 = pml2e_get_or_create_pml1(pml2e);
194 const pml1e_t *pml1e = pml1_entry(pml1, vaddr);
195 return pml1e_is_present(pml1e);
196}
197
198void *__create_page_table(void)
199{
200 mmstat_inc1(MEM_PAGETABLE);
201 return (void *) phyframe_va(mm_get_free_page());
202}
203
204void __destroy_page_table(void *table)
205{
206 mmstat_dec1(MEM_PAGETABLE);
207 pr_dinfo2(vmm, "__destroy_page_table: table=" PTR_FMT, (ptr_t) table);
208 mm_free_page(va_phyframe(table));
209}
210