1// SPDX-License-Identifier: GPL-3.0-or-later
2
3#include "mos/platform/platform_defs.hpp"
4
5#include <mos/lib/structures/list.hpp>
6#include <mos/lib/sync/spinlock.hpp>
7#include <mos/locks/futex.hpp>
8#include <mos/mm/paging/paging.hpp>
9#include <mos/platform/platform.hpp>
10#include <mos/syslog/printk.hpp>
11#include <mos/tasks/schedule.hpp>
12#include <mos/tasks/wait.hpp>
13#include <mos/type_utils.hpp>
14#include <mos/types.hpp>
15#include <mos_stdlib.hpp>
16
17typedef ptr_t futex_key_t;
18
19struct futex_private_t : mos::NamedType<"Futex.Private">
20{
21 as_linked_list;
22 futex_key_t key;
23 waitlist_t waiters;
24};
25
26static list_head futex_list_head;
27static spinlock_t futex_list_lock;
28
29static futex_key_t futex_get_key(const futex_word_t *futex)
30{
31 const ptr_t vaddr = (ptr_t) futex;
32 if (vaddr >= MOS_KERNEL_START_VADDR)
33 return vaddr;
34 return mm_get_phys_addr(current_process->mm, vaddr);
35}
36
37bool futex_wait(futex_word_t *fword, futex_word_t expected)
38{
39 const futex_word_t current_value = __atomic_load_n(fword, __ATOMIC_SEQ_CST);
40
41 if (current_value != expected)
42 {
43 //
44 // The purpose of the comparison with the expected value is to prevent lost wake-ups.
45 //
46 // if another thread changed the futex word value after the calling thread decided to block based on the prior value
47 // and, if that thread executed a futex_wake (or similar wake-up) after the value change before this FUTEX_WAIT operation
48 // then, with this check, the calling thread will observe the value change and will not start to sleep.
49 //
50 // | thread A | thread B |
51 // |--------------------|--------------------|
52 // | Check futex value | |
53 // | decide to block | |
54 // | | Change futex value |
55 // | | Execute futex_wake |
56 // | system call | |
57 // |--------------------|--------------------|
58 // | this check fails | | <--- if this check was not here, thread A would block, losing a wake-up
59 // |--------------------|--------------------|
60 // | unblocked | |
61 // |--------------------|--------------------|
62 //
63 return false;
64 }
65
66 // firstly find the futex in the list
67 // if it's not there, create a new one add the current thread to the waiters list
68 // then reschedule
69 const futex_key_t key = futex_get_key(futex: fword);
70
71 futex_private_t *fu = NULL;
72
73 spinlock_acquire(&futex_list_lock);
74 list_foreach(futex_private_t, f, futex_list_head)
75 {
76 if (f->key == key)
77 {
78 fu = f;
79 break;
80 }
81 }
82
83 if (!fu)
84 {
85 fu = mos::create<futex_private_t>();
86 fu->key = key;
87 waitlist_init(list: &fu->waiters);
88 list_node_append(head: &futex_list_head, list_node(fu));
89 }
90 spinlock_release(&futex_list_lock);
91
92 dInfo2<futex> << "tid " << current_thread << " waiting on lock key=" << key;
93
94 bool ok = reschedule_for_waitlist(waitlist: &fu->waiters);
95 MOS_ASSERT(ok);
96
97 dInfo2<futex> << "tid " << current_thread << " woke up";
98 return true;
99}
100
101bool futex_wake(futex_word_t *fword, size_t num_to_wake)
102{
103 if (unlikely(num_to_wake == 0))
104 mos_panic("insane number of threads to wake up (?): %zd", num_to_wake);
105
106 const futex_key_t key = futex_get_key(futex: fword);
107 futex_private_t *fu = NULL;
108
109 spinlock_acquire(&futex_list_lock);
110 list_foreach(futex_private_t, f, futex_list_head)
111 {
112 if (f->key == key)
113 {
114 fu = f;
115 break;
116 }
117 }
118 spinlock_release(&futex_list_lock);
119
120 if (!fu)
121 {
122 // no threads are waiting on this futex, or it's the first time it's being used by only one thread
123 return true;
124 }
125
126 dInfo2<futex> << "waking up " << num_to_wake << " threads on lock key=" << key;
127 const size_t real_wakeups = waitlist_wake(list: &fu->waiters, max_wakeups: num_to_wake);
128 dInfo2<futex> << "actually woke up " << real_wakeups << " threads";
129
130 return true;
131}
132