1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/mm.hpp"
4#include "mos/mm/paging/pml_types.hpp"
5#include "mos/mm/paging/pmlx/pml2.hpp"
6#include "mos/mm/paging/pmlx/pml3.hpp"
7#include "mos/mm/paging/pmlx/pml4.hpp"
8#include "mos/platform/platform.hpp"
9#include "mos/syslog/printk.hpp"
10#include "mos/x86/cpu/cpuid.hpp"
11#include "mos/x86/mm/paging_impl.hpp"
12#include "mos/x86/x86_platform.hpp"
13
14#include <algorithm>
15#include <mos/lib/structures/list.hpp>
16#include <mos/lib/sync/spinlock.hpp>
17#include <mos/types.hpp>
18#include <mos_stdlib.hpp>
19
20static void x86_setup_direct_map(pml4_t pml4)
21{
22 // map all memory to MOS_DIRECT_MAP_VADDR using 1 GiB, or 2 MiB pages
23 const bool gbpages = cpu_has_feature(CPU_FEATURE_PDPE1GB);
24
25 pr_dinfo2(x86_startup, "mapping all memory to " PTR_FMT " using %s pages", x86_platform.direct_map_base, gbpages ? "1 GB" : "2 MB");
26
27 const size_t STEP = (gbpages ? 1 GB : 2 MB) / MOS_PAGE_SIZE;
28 const size_t total_npages = std::max(ALIGN_UP(platform_info->max_pfn, STEP), b: 4 GB / MOS_PAGE_SIZE);
29
30 pfn_t pfn = 0;
31 while (pfn < total_npages)
32 {
33 const ptr_t vaddr = pfn_va(pfn);
34 pml4e_t *pml4e = pml4_entry(pml4, vaddr);
35 platform_pml4e_set_flags(pml4: pml4e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
36
37 if (gbpages)
38 {
39 // GB pages are at pml3e level
40 const pml3_t pml3 = pml4e_get_or_create_pml3(pml4e);
41 pml3e_t *pml3e = pml3_entry(pml3, vaddr);
42 platform_pml3e_set_huge(pml3e, pfn);
43 platform_pml3e_set_flags(pml3e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
44 }
45 else
46 {
47 // 2 MiB pages are at pml2e level
48 pml3e_t *pml3e = pml3_entry(pml3: pml4e_get_or_create_pml3(pml4e), vaddr);
49 platform_pml3e_set_flags(pml3e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
50
51 const pml2_t pml2 = pml3e_get_or_create_pml2(pml3e);
52 pml2e_t *pml2e = pml2_entry(pml2, vaddr);
53 platform_pml2e_set_huge(pml2: pml2e, pfn);
54 platform_pml2e_set_flags(pml2: pml2e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
55 }
56
57 pfn += STEP;
58 }
59}
60
61void x86_paging_setup()
62{
63 x86_setup_direct_map(pml4: platform_info->kernel_mm->pgd.max.next);
64}
65
66pfn_t platform_pml1e_get_pfn(const pml1e_t *pml1e)
67{
68 const x86_pte64_t *entry = cast<x86_pte64_t>(value: pml1e);
69 return entry->pfn;
70}
71
72void platform_pml1e_set_pfn(pml1e_t *pml1e, pfn_t pfn)
73{
74 x86_pte64_t *entry = cast<x86_pte64_t>(value: pml1e);
75 entry->present = true;
76 entry->pfn = pfn;
77}
78
79bool platform_pml1e_get_present(const pml1e_t *pml1e)
80{
81 const x86_pte64_t *entry = cast<x86_pte64_t>(value: pml1e);
82 return entry->present;
83}
84
85void platform_pml1e_set_flags(pml1e_t *pml1e, VMFlags flags)
86{
87 x86_pte64_t *entry = cast<x86_pte64_t>(value: pml1e);
88 entry->writable = flags & VM_WRITE;
89 entry->usermode = flags & VM_USER;
90 entry->write_through = flags & VM_WRITE_THROUGH;
91 entry->cache_disabled = flags & VM_CACHE_DISABLED;
92 entry->global = flags & VM_GLOBAL;
93 entry->no_execute = !(flags & VM_EXEC);
94}
95
96VMFlags platform_pml1e_get_flags(const pml1e_t *pml1e)
97{
98 const x86_pte64_t *entry = cast<x86_pte64_t>(value: pml1e);
99 VMFlags flags = VM_READ;
100 flags |= entry->writable ? VM_WRITE : VM_NONE;
101 flags |= entry->usermode ? VM_USER : VM_NONE;
102 flags |= entry->write_through ? VM_WRITE_THROUGH : VM_NONE;
103 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : VM_NONE;
104 flags |= entry->global ? VM_GLOBAL : VM_NONE;
105 flags |= entry->no_execute ? VM_NONE : VM_EXEC;
106 return flags;
107}
108
109// PML2
110
111pml1_t platform_pml2e_get_pml1(const pml2e_t *pml2e)
112{
113 const x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
114 return { .table = (pml1e_t *) pfn_va(entry->page_table_paddr) };
115}
116
117void platform_pml2e_set_pml1(pml2e_t *pml2e, pml1_t pml1, pfn_t pml1_pfn)
118{
119 MOS_UNUSED(pml1);
120 x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
121 entry->present = true;
122 entry->page_table_paddr = pml1_pfn;
123}
124
125bool platform_pml2e_get_present(const pml2e_t *pml2e)
126{
127 const x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
128 return entry->present;
129}
130
131void platform_pml2e_set_flags(pml2e_t *pml2e, VMFlags flags)
132{
133 x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
134 entry->writable |= flags & VM_WRITE;
135 entry->usermode |= flags & VM_USER;
136 entry->write_through |= flags & VM_WRITE_THROUGH;
137 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
138 if (flags & VM_EXEC)
139 entry->no_execute = false;
140
141 if (entry->page_size)
142 {
143 entry->no_execute = !(flags & VM_EXEC);
144 x86_pde64_huge_t *huge_entry = cast<x86_pde64_huge_t>(value: pml2e);
145 huge_entry->global = flags & VM_GLOBAL;
146 }
147}
148
149VMFlags platform_pml2e_get_flags(const pml2e_t *pml2e)
150{
151 const x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
152 VMFlags flags = VM_READ;
153 flags |= entry->writable ? VM_WRITE : VM_NONE;
154 flags |= entry->usermode ? VM_USER : VM_NONE;
155 flags |= entry->write_through ? VM_WRITE_THROUGH : VM_NONE;
156 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : VM_NONE;
157 flags |= entry->no_execute ? VM_NONE : VM_EXEC;
158 if (entry->page_size)
159 {
160 const x86_pde64_huge_t *huge_entry = cast<x86_pde64_huge_t>(value: pml2e);
161 flags |= huge_entry->global ? VM_GLOBAL : VM_NONE;
162 }
163
164 return flags;
165}
166
167bool platform_pml2e_is_huge(const pml2e_t *pml2e)
168{
169 const x86_pde64_t *entry = cast<x86_pde64_t>(value: pml2e);
170 return entry->page_size;
171}
172
173void platform_pml2e_set_huge(pml2e_t *pml2e, pfn_t pfn)
174{
175 pml2e->content = 0;
176 x86_pde64_huge_t *entry = cast<x86_pde64_huge_t>(value: pml2e);
177 entry->present = true;
178 entry->page_size = true;
179 entry->pfn = pfn >> 1; // 1 bit PAT
180}
181
182pfn_t platform_pml2e_get_huge_pfn(const pml2e_t *pml2)
183{
184 const x86_pde64_huge_t *entry = cast<x86_pde64_huge_t>(value: pml2);
185 return entry->pfn << 1; // 1 bit PAT
186}
187
188// PML3
189
190pml2_t platform_pml3e_get_pml2(const pml3e_t *pml3e)
191{
192 const x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
193 return { .table = (pml2e_t *) pfn_va(entry->page_table_paddr) };
194}
195
196void platform_pml3e_set_pml2(pml3e_t *pml3e, pml2_t pml2, pfn_t pml2_pfn)
197{
198 MOS_UNUSED(pml2);
199 x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
200 entry->present = true;
201 entry->page_table_paddr = pml2_pfn;
202}
203
204bool platform_pml3e_get_present(const pml3e_t *pml3e)
205{
206 const x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
207 return entry->present;
208}
209
210void platform_pml3e_set_flags(pml3e_t *pml3e, VMFlags flags)
211{
212 x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
213 entry->writable |= flags & VM_WRITE;
214 entry->usermode |= flags & VM_USER;
215 entry->write_through |= flags & VM_WRITE_THROUGH;
216 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
217 if (flags & VM_EXEC)
218 entry->no_execute = false; // if not huge, set NX bit only if exec flag is set
219
220 if (entry->page_size)
221 {
222 entry->no_execute = !(flags & VM_EXEC); // if huge, set NX bit accordingly
223 x86_pmde64_huge_t *huge_entry = cast<x86_pmde64_huge_t>(value: pml3e);
224 huge_entry->global = flags & VM_GLOBAL;
225 }
226}
227
228VMFlags platform_pml3e_get_flags(const pml3e_t *pml3e)
229{
230 const x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
231 VMFlags flags = VM_READ;
232 flags |= entry->writable ? VM_WRITE : VM_NONE;
233 flags |= entry->usermode ? VM_USER : VM_NONE;
234 flags |= entry->write_through ? VM_WRITE_THROUGH : VM_NONE;
235 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : VM_NONE;
236 flags |= entry->no_execute ? VM_NONE : VM_EXEC;
237 if (entry->page_size)
238 {
239 const x86_pmde64_huge_t *huge_entry = cast<x86_pmde64_huge_t>(value: pml3e);
240 flags |= huge_entry->global ? VM_GLOBAL : VM_NONE;
241 }
242 return flags;
243}
244
245bool platform_pml3e_is_huge(const pml3e_t *pml3e)
246{
247 const x86_pmde64_t *entry = cast<x86_pmde64_t>(value: pml3e);
248 return entry->page_size;
249}
250
251void platform_pml3e_set_huge(pml3e_t *pml3e, pfn_t pfn)
252{
253 pml3e->content = 0;
254 x86_pmde64_huge_t *entry = cast<x86_pmde64_huge_t>(value: pml3e);
255 entry->present = true;
256 entry->page_size = true;
257 entry->pfn = pfn >> 1; // 1 bit PAT
258}
259
260pfn_t platform_pml3e_get_huge_pfn(const pml3e_t *pml3)
261{
262 const x86_pmde64_huge_t *entry = cast<x86_pmde64_huge_t>(value: pml3);
263 return entry->pfn << 1; // 1 bit PAT
264}
265
266// PML4
267
268pml3_t platform_pml4e_get_pml3(const pml4e_t *pml4e)
269{
270 const x86_pude64_t *entry = cast<x86_pude64_t>(value: pml4e);
271 return { .table = (pml3e_t *) pfn_va(entry->page_table_paddr) };
272}
273
274void platform_pml4e_set_pml3(pml4e_t *pml4e, pml3_t pml3, pfn_t pml3_pfn)
275{
276 MOS_UNUSED(pml3);
277 x86_pude64_t *entry = cast<x86_pude64_t>(value: pml4e);
278 entry->present = true;
279 entry->page_table_paddr = pml3_pfn;
280}
281
282bool platform_pml4e_get_present(const pml4e_t *pml4e)
283{
284 const x86_pude64_t *entry = cast<x86_pude64_t>(value: pml4e);
285 return entry->present;
286}
287
288void platform_pml4e_set_flags(pml4e_t *pml4e, VMFlags flags)
289{
290 x86_pude64_t *entry = cast<x86_pude64_t>(value: pml4e);
291 entry->writable |= flags & VM_WRITE;
292 entry->usermode |= flags & VM_USER;
293 entry->write_through |= flags & VM_WRITE_THROUGH;
294 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
295 if (flags & VM_EXEC)
296 entry->no_execute = false;
297}
298
299VMFlags platform_pml4e_get_flags(const pml4e_t *pml4e)
300{
301 const x86_pude64_t *entry = cast<x86_pude64_t>(value: pml4e);
302 VMFlags flags = VM_READ;
303 if (entry->writable)
304 flags |= VM_WRITE;
305 flags |= entry->writable ? VM_WRITE : VM_NONE;
306 flags |= entry->usermode ? VM_USER : VM_NONE;
307 flags |= entry->write_through ? VM_WRITE_THROUGH : VM_NONE;
308 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : VM_NONE;
309 flags |= entry->no_execute ? VM_NONE : VM_EXEC;
310 return flags;
311}
312