1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/mm/cow.h"
4
5#include "mos/mm/mm.h"
6#include "mos/mm/mmstat.h"
7#include "mos/mm/paging/paging.h"
8#include "mos/platform/platform.h"
9
10#include <mos/interrupt/ipi.h>
11#include <mos/mm/cow.h>
12#include <mos/mm/paging/paging.h>
13#include <mos/mm/physical/pmm.h>
14#include <mos/platform/platform.h>
15#include <mos/syslog/printk.h>
16#include <mos/tasks/process.h>
17#include <mos/tasks/task_types.h>
18#include <mos/types.h>
19#include <mos_string.h>
20
21static phyframe_t *_zero_page = NULL;
22static phyframe_t *zero_page(void)
23{
24 if (unlikely(!_zero_page))
25 {
26 _zero_page = pmm_ref_one(mm_get_free_page());
27 MOS_ASSERT(_zero_page);
28 memzero(s: (void *) phyframe_va(_zero_page), MOS_PAGE_SIZE);
29 }
30
31 return _zero_page;
32}
33
34static vmfault_result_t cow_zod_fault_handler(vmap_t *vmap, ptr_t fault_addr, pagefault_t *info)
35{
36 MOS_UNUSED(fault_addr);
37
38 if (info->is_present && info->is_write)
39 {
40 vmap_stat_dec(vmap, cow); // the faulting page is a CoW page
41 vmap_stat_inc(vmap, regular);
42 return mm_resolve_cow_fault(vmap, fault_addr, info);
43 }
44
45 MOS_ASSERT(!info->is_present); // we can't have (present && !write)
46
47 if (info->is_write)
48 {
49 // non-present and write, must be a ZoD page
50 info->backing_page = mm_get_free_page();
51 vmap_stat_inc(vmap, regular);
52 return VMFAULT_MAP_BACKING_PAGE;
53 }
54 else
55 {
56 info->backing_page = zero_page();
57 vmap_stat_inc(vmap, cow);
58 return VMFAULT_MAP_BACKING_PAGE_RO;
59 }
60}
61
62vmap_t *cow_clone_vmap_locked(mm_context_t *target_mmctx, vmap_t *src_vmap)
63{
64 // remove that VM_WRITE flag
65 mm_flag_pages_locked(mmctx: src_vmap->mmctx, vaddr: src_vmap->vaddr, npages: src_vmap->npages, flags: src_vmap->vmflags & ~VM_WRITE);
66 src_vmap->stat.cow += src_vmap->stat.regular;
67 src_vmap->stat.regular = 0; // no longer private
68
69 vmap_t *dst_vmap = mm_clone_vmap_locked(src_vmap, dst_ctx: target_mmctx);
70
71 if (!src_vmap->on_fault)
72 src_vmap->on_fault = cow_zod_fault_handler;
73
74 dst_vmap->on_fault = src_vmap->on_fault;
75 dst_vmap->stat = dst_vmap->stat;
76 return dst_vmap;
77}
78
79vmap_t *cow_allocate_zeroed_pages(mm_context_t *mmctx, size_t npages, ptr_t vaddr, valloc_flags allocflags, vm_flags flags)
80{
81 spinlock_acquire(&mmctx->mm_lock);
82 vmap_t *vmap = mm_get_free_vaddr_locked(mmctx, n_pages: npages, base_vaddr: vaddr, flags: allocflags);
83 spinlock_release(&mmctx->mm_lock);
84
85 if (IS_ERR(ptr: vmap))
86 return vmap;
87
88 vmap->vmflags = flags;
89 vmap->on_fault = cow_zod_fault_handler;
90 return vmap;
91}
92