1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/mm.h"
4#include "mos/mm/paging/pml_types.h"
5#include "mos/mm/paging/pmlx/pml2.h"
6#include "mos/mm/paging/pmlx/pml3.h"
7#include "mos/mm/paging/pmlx/pml4.h"
8#include "mos/platform/platform.h"
9#include "mos/syslog/printk.h"
10#include "mos/x86/cpu/cpuid.h"
11#include "mos/x86/mm/paging_impl.h"
12#include "mos/x86/x86_platform.h"
13
14#include <mos/lib/structures/list.h>
15#include <mos/lib/sync/spinlock.h>
16#include <mos/types.h>
17#include <mos_stdlib.h>
18
19static void x86_setup_direct_map(pml4_t pml4)
20{
21 // map all memory to MOS_DIRECT_MAP_VADDR using 1 GiB, or 2 MiB pages
22 const bool gbpages = cpu_has_feature(CPU_FEATURE_PDPE1GB);
23
24 pr_dinfo2(x86_startup, "mapping all memory to " PTR_FMT " using %s pages", x86_platform.direct_map_base, gbpages ? "1 GB" : "2 MB");
25
26 const size_t STEP = (gbpages ? 1 GB : 2 MB) / MOS_PAGE_SIZE;
27 const size_t total_npages = MAX(ALIGN_UP(platform_info->max_pfn, STEP), 4 GB / MOS_PAGE_SIZE);
28
29 pfn_t pfn = 0;
30 while (pfn < total_npages)
31 {
32 const ptr_t vaddr = pfn_va(pfn);
33 pml4e_t *pml4e = pml4_entry(pml4, vaddr);
34 platform_pml4e_set_flags(pml4: pml4e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
35
36 if (gbpages)
37 {
38 // GB pages are at pml3e level
39 const pml3_t pml3 = pml4e_get_or_create_pml3(pml4e);
40 pml3e_t *pml3e = pml3_entry(pml3, vaddr);
41 platform_pml3e_set_huge(pml3: pml3e, pfn);
42 platform_pml3e_set_flags(pml3: pml3e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
43 }
44 else
45 {
46 // 2 MiB pages are at pml2e level
47 pml3e_t *pml3e = pml3_entry(pml3: pml4e_get_or_create_pml3(pml4e), vaddr);
48 platform_pml3e_set_flags(pml3: pml3e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
49
50 const pml2_t pml2 = pml3e_get_or_create_pml2(pml3e);
51 pml2e_t *pml2e = pml2_entry(pml2, vaddr);
52 platform_pml2e_set_huge(pml2: pml2e, pfn);
53 platform_pml2e_set_flags(pml2: pml2e, flags: VM_READ | VM_WRITE | VM_GLOBAL);
54 }
55
56 pfn += STEP;
57 }
58}
59
60void x86_paging_setup()
61{
62 x86_setup_direct_map(pml4: platform_info->kernel_mm->pgd.max.next);
63}
64
65pfn_t platform_pml1e_get_pfn(const pml1e_t *pml1e)
66{
67 const x86_pte64_t *entry = cast_to(pml1e, pml1e_t *, x86_pte64_t *);
68 return entry->pfn;
69}
70
71void platform_pml1e_set_pfn(pml1e_t *pml1e, pfn_t pfn)
72{
73 x86_pte64_t *entry = cast_to(pml1e, pml1e_t *, x86_pte64_t *);
74 entry->present = true;
75 entry->pfn = pfn;
76}
77
78bool platform_pml1e_get_present(const pml1e_t *pml1e)
79{
80 const x86_pte64_t *entry = cast_to(pml1e, pml1e_t *, x86_pte64_t *);
81 return entry->present;
82}
83
84void platform_pml1e_set_flags(pml1e_t *pml1e, vm_flags flags)
85{
86 x86_pte64_t *entry = cast_to(pml1e, pml1e_t *, x86_pte64_t *);
87 entry->writable = flags & VM_WRITE;
88 entry->usermode = flags & VM_USER;
89 entry->write_through = flags & VM_WRITE_THROUGH;
90 entry->cache_disabled = flags & VM_CACHE_DISABLED;
91 entry->global = flags & VM_GLOBAL;
92 entry->no_execute = !(flags & VM_EXEC);
93}
94
95vm_flags platform_pml1e_get_flags(const pml1e_t *pml1e)
96{
97 const x86_pte64_t *entry = cast_to(pml1e, pml1e_t *, x86_pte64_t *);
98 vm_flags flags = VM_READ;
99 flags |= entry->writable ? VM_WRITE : 0;
100 flags |= entry->usermode ? VM_USER : 0;
101 flags |= entry->write_through ? VM_WRITE_THROUGH : 0;
102 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : 0;
103 flags |= entry->global ? VM_GLOBAL : 0;
104 flags |= entry->no_execute ? 0 : VM_EXEC;
105 return flags;
106}
107
108// PML2
109
110pml1_t platform_pml2e_get_pml1(const pml2e_t *pml2e)
111{
112 const x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
113 return (pml1_t){ .table = (pml1e_t *) pfn_va(entry->page_table_paddr) };
114}
115
116void platform_pml2e_set_pml1(pml2e_t *pml2e, pml1_t pml1, pfn_t pml1_pfn)
117{
118 MOS_UNUSED(pml1);
119 x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
120 entry->present = true;
121 entry->page_table_paddr = pml1_pfn;
122}
123
124bool platform_pml2e_get_present(const pml2e_t *pml2e)
125{
126 const x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
127 return entry->present;
128}
129
130void platform_pml2e_set_flags(pml2e_t *pml2e, vm_flags flags)
131{
132 x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
133 entry->writable |= flags & VM_WRITE;
134 entry->usermode |= flags & VM_USER;
135 entry->write_through |= flags & VM_WRITE_THROUGH;
136 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
137 if (flags & VM_EXEC)
138 entry->no_execute = false;
139
140 if (entry->page_size)
141 {
142 entry->no_execute = !(flags & VM_EXEC);
143 x86_pde64_huge_t *huge_entry = cast_to(entry, x86_pde64_t *, x86_pde64_huge_t *);
144 huge_entry->global = flags & VM_GLOBAL;
145 }
146}
147
148vm_flags platform_pml2e_get_flags(const pml2e_t *pml2e)
149{
150 const x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
151 vm_flags flags = VM_READ;
152 flags |= entry->writable ? VM_WRITE : 0;
153 flags |= entry->usermode ? VM_USER : 0;
154 flags |= entry->write_through ? VM_WRITE_THROUGH : 0;
155 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : 0;
156 flags |= entry->no_execute ? 0 : VM_EXEC;
157 if (entry->page_size)
158 {
159 const x86_pde64_huge_t *huge_entry = cast_to(pml2e, pml2e_t *, x86_pde64_huge_t *);
160 flags |= huge_entry->global ? VM_GLOBAL : 0;
161 }
162
163 return flags;
164}
165
166bool platform_pml2e_is_huge(const pml2e_t *pml2e)
167{
168 const x86_pde64_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_t *);
169 return entry->page_size;
170}
171
172void platform_pml2e_set_huge(pml2e_t *pml2e, pfn_t pfn)
173{
174 pml2e->content = 0;
175 x86_pde64_huge_t *entry = cast_to(pml2e, pml2e_t *, x86_pde64_huge_t *);
176 entry->present = true;
177 entry->page_size = true;
178 entry->pfn = pfn >> 1; // 1 bit PAT
179}
180
181pfn_t platform_pml2e_get_huge_pfn(const pml2e_t *pml2)
182{
183 const x86_pde64_huge_t *entry = cast_to(pml2, pml2e_t *, x86_pde64_huge_t *);
184 return entry->pfn << 1; // 1 bit PAT
185}
186
187// PML3
188
189pml2_t platform_pml3e_get_pml2(const pml3e_t *pml3e)
190{
191 const x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
192 return (pml2_t){ .table = (pml2e_t *) pfn_va(entry->page_table_paddr) };
193}
194
195void platform_pml3e_set_pml2(pml3e_t *pml3e, pml2_t pml2, pfn_t pml2_pfn)
196{
197 MOS_UNUSED(pml2);
198 x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
199 entry->present = true;
200 entry->page_table_paddr = pml2_pfn;
201}
202
203bool platform_pml3e_get_present(const pml3e_t *pml3e)
204{
205 const x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
206 return entry->present;
207}
208
209void platform_pml3e_set_flags(pml3e_t *pml3e, vm_flags flags)
210{
211 x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
212 entry->writable |= flags & VM_WRITE;
213 entry->usermode |= flags & VM_USER;
214 entry->write_through |= flags & VM_WRITE_THROUGH;
215 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
216 if (flags & VM_EXEC)
217 entry->no_execute = false; // if not huge, set NX bit only if exec flag is set
218
219 if (entry->page_size)
220 {
221 entry->no_execute = !(flags & VM_EXEC); // if huge, set NX bit accordingly
222 x86_pmde64_huge_t *huge_entry = cast_to(entry, x86_pmde64_t *, x86_pmde64_huge_t *);
223 huge_entry->global = flags & VM_GLOBAL;
224 }
225}
226
227vm_flags platform_pml3e_get_flags(const pml3e_t *pml3e)
228{
229 const x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
230 vm_flags flags = VM_READ;
231 flags |= entry->writable ? VM_WRITE : 0;
232 flags |= entry->usermode ? VM_USER : 0;
233 flags |= entry->write_through ? VM_WRITE_THROUGH : 0;
234 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : 0;
235 flags |= entry->no_execute ? 0 : VM_EXEC;
236 if (entry->page_size)
237 {
238 const x86_pmde64_huge_t *huge_entry = cast_to(pml3e, pml3e_t *, x86_pmde64_huge_t *);
239 flags |= huge_entry->global ? VM_GLOBAL : 0;
240 }
241 return flags;
242}
243
244bool platform_pml3e_is_huge(const pml3e_t *pml3e)
245{
246 const x86_pmde64_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_t *);
247 return entry->page_size;
248}
249
250void platform_pml3e_set_huge(pml3e_t *pml3e, pfn_t pfn)
251{
252 pml3e->content = 0;
253 x86_pmde64_huge_t *entry = cast_to(pml3e, pml3e_t *, x86_pmde64_huge_t *);
254 entry->present = true;
255 entry->page_size = true;
256 entry->pfn = pfn >> 1; // 1 bit PAT
257}
258
259pfn_t platform_pml3e_get_huge_pfn(const pml3e_t *pml3)
260{
261 const x86_pmde64_huge_t *entry = cast_to(pml3, pml3e_t *, x86_pmde64_huge_t *);
262 return entry->pfn << 1; // 1 bit PAT
263}
264
265// PML4
266
267pml3_t platform_pml4e_get_pml3(const pml4e_t *pml4e)
268{
269 const x86_pude64_t *entry = cast_to(pml4e, pml4e_t *, x86_pude64_t *);
270 return (pml3_t){ .table = (pml3e_t *) pfn_va(entry->page_table_paddr) };
271}
272
273void platform_pml4e_set_pml3(pml4e_t *pml4e, pml3_t pml3, pfn_t pml3_pfn)
274{
275 MOS_UNUSED(pml3);
276 x86_pude64_t *entry = cast_to(pml4e, pml4e_t *, x86_pude64_t *);
277 entry->present = true;
278 entry->page_table_paddr = pml3_pfn;
279}
280
281bool platform_pml4e_get_present(const pml4e_t *pml4e)
282{
283 const x86_pude64_t *entry = cast_to(pml4e, pml4e_t *, x86_pude64_t *);
284 return entry->present;
285}
286
287void platform_pml4e_set_flags(pml4e_t *pml4e, vm_flags flags)
288{
289 x86_pude64_t *entry = cast_to(pml4e, pml4e_t *, x86_pude64_t *);
290 entry->writable |= flags & VM_WRITE;
291 entry->usermode |= flags & VM_USER;
292 entry->write_through |= flags & VM_WRITE_THROUGH;
293 entry->cache_disabled |= flags & VM_CACHE_DISABLED;
294 if (flags & VM_EXEC)
295 entry->no_execute = false;
296}
297
298vm_flags platform_pml4e_get_flags(const pml4e_t *pml4e)
299{
300 const x86_pude64_t *entry = cast_to(pml4e, pml4e_t *, x86_pude64_t *);
301 vm_flags flags = VM_READ;
302 flags |= entry->writable ? VM_WRITE : 0;
303 flags |= entry->usermode ? VM_USER : 0;
304 flags |= entry->write_through ? VM_WRITE_THROUGH : 0;
305 flags |= entry->cache_disabled ? VM_CACHE_DISABLED : 0;
306 flags |= entry->no_execute ? 0 : VM_EXEC;
307 return flags;
308}
309