1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/paging/iterator.h"
4
5#include "mos/mm/paging/pmlx/pml1.h"
6#include "mos/mm/paging/pmlx/pml2.h"
7#include "mos/mm/paging/pmlx/pml3.h"
8#include "mos/mm/paging/pmlx/pml4.h"
9#include "mos/platform/platform.h"
10
11#include <mos_string.h>
12
13static void pagetable_iterator_start_current_range(pagetable_iter_t *it, pml5_t *pml5, pml4_t *pml4, pml3_t *pml3, pml2_t *pml2, pml1_t *pml1)
14{
15 it->range = (pagetable_iter_range_t){ 0 };
16 it->range.vaddr = it->vaddr;
17 *pml5 = it->pgd.max;
18
19 *pml4 = pml5->next;
20 const pml4e_t *pml4e = pml4_entry(*pml4, it->vaddr);
21 if (!pml4e_is_present(pml4e))
22 return;
23
24#if MOS_CONFIG(PML4_HUGE_CAPABLE)
25 if (platform_pml4e_is_huge(pml4e))
26 {
27 it->range.present = true;
28 it->range.flags = platform_pml4e_get_flags(pml4e);
29 it->range.pfn = platform_pml4e_get_huge_pfn(pml4e);
30 it->vaddr += PML4E_NPAGES * MOS_PAGE_SIZE;
31 it->range.pfn_end = it->range.pfn + PML4E_NPAGES - 1;
32 return;
33 }
34#endif
35
36 *pml3 = platform_pml4e_get_pml3(pml4e);
37 const pml3e_t *pml3e = pml3_entry(*pml3, it->vaddr);
38 if (!pml3e_is_present(pml3e))
39 return;
40
41#if MOS_CONFIG(PML3_HUGE_CAPABLE)
42 if (platform_pml3e_is_huge(pml3e))
43 {
44 it->range.present = true;
45 it->range.flags = platform_pml3e_get_flags(pml3e);
46 it->range.pfn = platform_pml3e_get_huge_pfn(pml3e);
47 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
48 it->range.pfn_end = it->range.pfn + PML3E_NPAGES - 1;
49 return;
50 }
51#endif
52
53 *pml2 = platform_pml3e_get_pml2(pml3e);
54 const pml2e_t *pml2e = pml2_entry(*pml2, it->vaddr);
55 if (!pml2e_is_present(pml2e))
56 return;
57
58#if MOS_CONFIG(PML2_HUGE_CAPABLE)
59 if (platform_pml2e_is_huge(pml2e))
60 {
61 it->range.present = true;
62 it->range.flags = platform_pml2e_get_flags(pml2e);
63 it->range.pfn = platform_pml2e_get_huge_pfn(pml2e);
64 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
65 it->range.pfn_end = it->range.pfn + PML2E_NPAGES - 1;
66 return;
67 }
68#endif
69
70 *pml1 = platform_pml2e_get_pml1(pml2e);
71 const pml1e_t *pml1e = pml1_entry(*pml1, it->vaddr);
72 it->range.present = platform_pml1e_get_present(pml1e);
73 if (!it->range.present)
74 return;
75
76 it->range.flags = platform_pml1e_get_flags(pml1e);
77 it->range.pfn = platform_pml1e_get_pfn(pml1e);
78 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE;
79 it->range.pfn_end = it->range.pfn + PML1E_NPAGES - 1;
80}
81
82void pagetable_iter_init(pagetable_iter_t *it, pgd_t pgd, ptr_t vaddr, ptr_t end)
83{
84 memzero(s: it, n: sizeof(*it));
85 it->pgd = pgd;
86 it->vaddr = it->start = vaddr;
87 it->end = end;
88}
89
90#define yield_range() \
91 do \
92 { \
93 it->range.vaddr_end = it->vaddr - 1; \
94 return &it->range; \
95 } while (0)
96
97pagetable_iter_range_t *pagetable_iter_next(pagetable_iter_t *it)
98{
99#define _IS_IN_RANGE MOS_IN_RANGE(it->vaddr, it->start, it->end)
100 if (!_IS_IN_RANGE)
101 return NULL;
102
103 pml5_t pml5 = { 0 };
104 pml4_t pml4 = { 0 };
105 pml3_t pml3 = { 0 };
106 pml2_t pml2 = { 0 };
107 pml1_t pml1 = { 0 };
108
109 pagetable_iterator_start_current_range(it, pml5: &pml5, pml4: &pml4, pml3: &pml3, pml2: &pml2, pml1: &pml1);
110
111 if (!pml_null(pml1))
112 goto iterate_pml1;
113 else if (!pml_null(pml2))
114 goto iterate_pml2;
115 else if (!pml_null(pml3))
116 goto iterate_pml3;
117 else if (!pml_null(pml4))
118 goto iterate_pml4;
119 else
120 MOS_UNREACHABLE();
121
122 __builtin_unreachable();
123
124iterate_pml1:
125 for (u32 i = pml1_index(it->vaddr); i < PML1_ENTRIES && _IS_IN_RANGE; i++)
126 {
127 const pml1e_t *pml1e = &pml1.table[i];
128 const bool present = platform_pml1e_get_present(pml1: pml1e);
129
130 if (!present && present != it->range.present)
131 {
132 it->range.vaddr_end = it->vaddr - 1;
133 return &it->range;
134 }
135
136 if (!present)
137 {
138 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE;
139 continue;
140 }
141
142 const vm_flags new_flags = platform_pml1e_get_flags(pml1e);
143 const pfn_t new_pfn = platform_pml1e_get_pfn(pml1: pml1e);
144
145 bool changed = false;
146 changed |= new_flags != it->range.flags; // flags changed
147 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
148
149 if (changed)
150 yield_range();
151
152 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE; // next time, we start from the next page
153 it->range.pfn_end = new_pfn + PML1E_NPAGES - 1;
154 }
155
156iterate_pml2:
157 for (u32 i = pml2_index(it->vaddr); i < PML2_ENTRIES && _IS_IN_RANGE; i++)
158 {
159 const pml2e_t *pml2e = &pml2.table[i];
160 const bool present = pml2e_is_present(pml2e);
161
162 if (!present && present != it->range.present)
163 yield_range();
164
165 if (!present)
166 {
167 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
168 continue;
169 }
170
171#if MOS_CONFIG(PML2_HUGE_CAPABLE)
172 if (platform_pml2e_is_huge(pml2: pml2e))
173 {
174 const vm_flags new_flags = platform_pml2e_get_flags(pml2e);
175 const pfn_t new_pfn = platform_pml2e_get_huge_pfn(pml2: pml2e);
176
177 bool changed = false;
178 changed |= new_flags != it->range.flags; // flags changed
179 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
180
181 if (changed)
182 yield_range();
183
184 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
185 it->range.pfn_end = new_pfn + PML2E_NPAGES - 1;
186 }
187 else
188#endif
189 {
190 pml1 = platform_pml2e_get_pml1(pml2: pml2e);
191 goto iterate_pml1; // iterate pml1
192 }
193 }
194
195iterate_pml3:
196 for (u32 i = pml3_index(it->vaddr); i < PML3_ENTRIES && _IS_IN_RANGE; i++)
197 {
198 const pml3e_t *pml3e = pml3_entry(pml3, vaddr: it->vaddr);
199 const bool present = pml3e_is_present(pml3e);
200
201 if (!present && present != it->range.present)
202 yield_range();
203
204 if (!present)
205 {
206 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
207 continue;
208 }
209
210#if MOS_CONFIG(PML3_HUGE_CAPABLE)
211 if (platform_pml3e_is_huge(pml3: pml3e))
212 {
213 const vm_flags new_flags = platform_pml3e_get_flags(pml3e);
214 const pfn_t new_pfn = platform_pml3e_get_huge_pfn(pml3: pml3e);
215
216 bool changed = false;
217 changed |= new_flags != it->range.flags; // flags changed
218 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
219
220 if (changed)
221 yield_range();
222
223 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
224 it->range.pfn_end = new_pfn + PML3E_NPAGES - 1;
225 }
226 else
227#endif
228 {
229 pml2 = platform_pml3e_get_pml2(pml3: pml3e);
230 goto iterate_pml2; // iterate pml2
231 }
232 }
233
234iterate_pml4:
235 for (u32 i = pml4_index(it->vaddr); i < PML4_ENTRIES && _IS_IN_RANGE; i++)
236 {
237 const pml4e_t *pml4e = pml4_entry(pml4, vaddr: it->vaddr);
238 const bool present = pml4e_is_present(pml4e);
239
240 if (!present && present != it->range.present)
241 yield_range();
242
243 if (!present)
244 {
245 it->vaddr += PML4E_NPAGES * MOS_PAGE_SIZE;
246 continue;
247 }
248
249 pml3 = platform_pml4e_get_pml3(pml4: pml4e);
250 goto iterate_pml3; // iterate pml3
251 }
252
253 yield_range();
254}
255