1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/paging/iterator.hpp"
4
5#include "mos/assert.hpp"
6#include "mos/mm/paging/pmlx/pml1.hpp"
7#include "mos/mm/paging/pmlx/pml2.hpp"
8#include "mos/mm/paging/pmlx/pml3.hpp"
9#include "mos/mm/paging/pmlx/pml4.hpp"
10#include "mos/platform/platform.hpp"
11
12#include <mos_string.hpp>
13
14static void pagetable_iterator_start_current_range(pagetable_iter_t *it, pml5_t *pml5, pml4_t *pml4, pml3_t *pml3, pml2_t *pml2, pml1_t *pml1)
15{
16 it->range = (pagetable_iter_range_t) { .present: 0 };
17 it->range.vaddr = it->vaddr;
18 *pml5 = it->pgd.max;
19
20 *pml4 = pml5->next;
21 const pml4e_t *pml4e = pml4_entry(pml4: *pml4, vaddr: it->vaddr);
22 if (!pml4e_is_present(pml4e))
23 return;
24
25#if MOS_CONFIG(PML4_HUGE_CAPABLE)
26 if (platform_pml4e_is_huge(pml4e))
27 {
28 it->range.present = true;
29 it->range.flags = platform_pml4e_get_flags(pml4e);
30 it->range.pfn = platform_pml4e_get_huge_pfn(pml4e);
31 it->vaddr += PML4E_NPAGES * MOS_PAGE_SIZE;
32 it->range.pfn_end = it->range.pfn + PML4E_NPAGES - 1;
33 return;
34 }
35#endif
36
37 *pml3 = platform_pml4e_get_pml3(pml4: pml4e);
38 const pml3e_t *pml3e = pml3_entry(pml3: *pml3, vaddr: it->vaddr);
39 if (!pml3e_is_present(pml3e))
40 return;
41
42#if MOS_CONFIG(PML3_HUGE_CAPABLE)
43 if (platform_pml3e_is_huge(pml3e))
44 {
45 it->range.present = true;
46 it->range.flags = platform_pml3e_get_flags(pml3e);
47 it->range.pfn = platform_pml3e_get_huge_pfn(pml3e);
48 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
49 it->range.pfn_end = it->range.pfn + PML3E_NPAGES - 1;
50 return;
51 }
52#endif
53
54 *pml2 = platform_pml3e_get_pml2(pml3e);
55 const pml2e_t *pml2e = pml2_entry(pml2: *pml2, vaddr: it->vaddr);
56 if (!pml2e_is_present(pml2e))
57 return;
58
59#if MOS_CONFIG(PML2_HUGE_CAPABLE)
60 if (platform_pml2e_is_huge(pml2: pml2e))
61 {
62 it->range.present = true;
63 it->range.flags = platform_pml2e_get_flags(pml2e);
64 it->range.pfn = platform_pml2e_get_huge_pfn(pml2: pml2e);
65 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
66 it->range.pfn_end = it->range.pfn + PML2E_NPAGES - 1;
67 return;
68 }
69#endif
70
71 *pml1 = platform_pml2e_get_pml1(pml2: pml2e);
72 const pml1e_t *pml1e = pml1_entry(pml1: *pml1, vaddr: it->vaddr);
73 it->range.present = platform_pml1e_get_present(pml1: pml1e);
74 if (!it->range.present)
75 return;
76
77 it->range.flags = platform_pml1e_get_flags(pml1e);
78 it->range.pfn = platform_pml1e_get_pfn(pml1: pml1e);
79 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE;
80 it->range.pfn_end = it->range.pfn + PML1E_NPAGES - 1;
81}
82
83void pagetable_iter_init(pagetable_iter_t *it, pgd_t pgd, ptr_t vaddr, ptr_t end)
84{
85 memzero(s: it, n: sizeof(*it));
86 it->pgd = pgd;
87 it->vaddr = it->start = vaddr;
88 it->end = end;
89}
90
91#define yield_range() \
92 do \
93 { \
94 it->range.vaddr_end = it->vaddr - 1; \
95 return &it->range; \
96 } while (0)
97
98pagetable_iter_range_t *pagetable_iter_next(pagetable_iter_t *it)
99{
100#define _IS_IN_RANGE MOS_IN_RANGE(it->vaddr, it->start, it->end)
101 if (!_IS_IN_RANGE)
102 return NULL;
103
104 pml5_t pml5 = { .next.table: 0 };
105 pml4_t pml4 = { .table: 0 };
106 pml3_t pml3 = { .table: 0 };
107 pml2_t pml2 = { .table: 0 };
108 pml1_t pml1 = { .table: 0 };
109
110 pagetable_iterator_start_current_range(it, pml5: &pml5, pml4: &pml4, pml3: &pml3, pml2: &pml2, pml1: &pml1);
111
112 if (!pml_null(pml1))
113 goto iterate_pml1;
114 else if (!pml_null(pml2))
115 goto iterate_pml2;
116 else if (!pml_null(pml3))
117 goto iterate_pml3;
118 else if (!pml_null(pml4))
119 goto iterate_pml4;
120 else
121 MOS_UNREACHABLE();
122
123 __builtin_unreachable();
124
125iterate_pml1:
126 for (u32 i = pml1_index(it->vaddr); i < PML1_ENTRIES && _IS_IN_RANGE; i++)
127 {
128 const pml1e_t *pml1e = &pml1.table[i];
129 const bool present = platform_pml1e_get_present(pml1: pml1e);
130
131 if (!present && present != it->range.present)
132 {
133 it->range.vaddr_end = it->vaddr - 1;
134 return &it->range;
135 }
136
137 if (!present)
138 {
139 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE;
140 continue;
141 }
142
143 const VMFlags new_flags = platform_pml1e_get_flags(pml1e);
144 const pfn_t new_pfn = platform_pml1e_get_pfn(pml1: pml1e);
145
146 bool changed = false;
147 changed |= new_flags != it->range.flags; // flags changed
148 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
149
150 if (changed)
151 yield_range();
152
153 it->vaddr += PML1E_NPAGES * MOS_PAGE_SIZE; // next time, we start from the next page
154 it->range.pfn_end = new_pfn + PML1E_NPAGES - 1;
155 }
156
157iterate_pml2:
158 for (u32 i = pml2_index(it->vaddr); i < PML2_ENTRIES && _IS_IN_RANGE; i++)
159 {
160 const pml2e_t *pml2e = &pml2.table[i];
161 const bool present = pml2e_is_present(pml2e);
162
163 if (!present && present != it->range.present)
164 yield_range();
165
166 if (!present)
167 {
168 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
169 continue;
170 }
171
172#if MOS_CONFIG(PML2_HUGE_CAPABLE)
173 if (platform_pml2e_is_huge(pml2: pml2e))
174 {
175 const VMFlags new_flags = platform_pml2e_get_flags(pml2e);
176 const pfn_t new_pfn = platform_pml2e_get_huge_pfn(pml2: pml2e);
177
178 bool changed = false;
179 changed |= new_flags != it->range.flags; // flags changed
180 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
181
182 if (changed)
183 yield_range();
184
185 it->vaddr += PML2E_NPAGES * MOS_PAGE_SIZE;
186 it->range.pfn_end = new_pfn + PML2E_NPAGES - 1;
187 }
188 else
189#endif
190 {
191 pml1 = platform_pml2e_get_pml1(pml2: pml2e);
192 goto iterate_pml1; // iterate pml1
193 }
194 }
195
196iterate_pml3:
197 for (u32 i = pml3_index(it->vaddr); i < PML3_ENTRIES && _IS_IN_RANGE; i++)
198 {
199 const pml3e_t *pml3e = pml3_entry(pml3, vaddr: it->vaddr);
200 const bool present = pml3e_is_present(pml3e);
201
202 if (!present && present != it->range.present)
203 yield_range();
204
205 if (!present)
206 {
207 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
208 continue;
209 }
210
211#if MOS_CONFIG(PML3_HUGE_CAPABLE)
212 if (platform_pml3e_is_huge(pml3e))
213 {
214 const VMFlags new_flags = platform_pml3e_get_flags(pml3e);
215 const pfn_t new_pfn = platform_pml3e_get_huge_pfn(pml3e);
216
217 bool changed = false;
218 changed |= new_flags != it->range.flags; // flags changed
219 changed |= new_pfn != it->range.pfn_end + 1; // pfn changed
220
221 if (changed)
222 yield_range();
223
224 it->vaddr += PML3E_NPAGES * MOS_PAGE_SIZE;
225 it->range.pfn_end = new_pfn + PML3E_NPAGES - 1;
226 }
227 else
228#endif
229 {
230 pml2 = platform_pml3e_get_pml2(pml3e);
231 goto iterate_pml2; // iterate pml2
232 }
233 }
234
235iterate_pml4:
236 for (u32 i = pml4_index(it->vaddr); i < PML4_ENTRIES && _IS_IN_RANGE; i++)
237 {
238 const pml4e_t *pml4e = pml4_entry(pml4, vaddr: it->vaddr);
239 const bool present = pml4e_is_present(pml4e);
240
241 if (!present && present != it->range.present)
242 yield_range();
243
244 if (!present)
245 {
246 it->vaddr += PML4E_NPAGES * MOS_PAGE_SIZE;
247 continue;
248 }
249
250 pml3 = platform_pml4e_get_pml3(pml4: pml4e);
251 goto iterate_pml3; // iterate pml3
252 }
253
254 yield_range();
255}
256