Branch data Line data Source code
1 : : /*
2 : : * Copyright (c) 2016-2017 Wind River Systems, Inc.
3 : : *
4 : : * SPDX-License-Identifier: Apache-2.0
5 : : */
6 : :
7 : : #ifndef ZEPHYR_KERNEL_INCLUDE_KSCHED_H_
8 : : #define ZEPHYR_KERNEL_INCLUDE_KSCHED_H_
9 : :
10 : : #include <zephyr/kernel_structs.h>
11 : : #include <kernel_internal.h>
12 : : #include <timeout_q.h>
13 : : #include <kthread.h>
14 : : #include <zephyr/tracing/tracing.h>
15 : : #include <stdbool.h>
16 : : #include <priority_q.h>
17 : :
18 : : BUILD_ASSERT(K_LOWEST_APPLICATION_THREAD_PRIO
19 : : >= K_HIGHEST_APPLICATION_THREAD_PRIO);
20 : :
21 : : #ifdef CONFIG_MULTITHREADING
22 : : #define Z_VALID_PRIO(prio, entry_point) \
23 : : (((prio) == K_IDLE_PRIO && z_is_idle_thread_entry(entry_point)) || \
24 : : ((K_LOWEST_APPLICATION_THREAD_PRIO \
25 : : >= K_HIGHEST_APPLICATION_THREAD_PRIO) \
26 : : && (prio) >= K_HIGHEST_APPLICATION_THREAD_PRIO \
27 : : && (prio) <= K_LOWEST_APPLICATION_THREAD_PRIO))
28 : :
29 : : #define Z_ASSERT_VALID_PRIO(prio, entry_point) do { \
30 : : __ASSERT(Z_VALID_PRIO((prio), (entry_point)), \
31 : : "invalid priority (%d); allowed range: %d to %d", \
32 : : (prio), \
33 : : K_LOWEST_APPLICATION_THREAD_PRIO, \
34 : : K_HIGHEST_APPLICATION_THREAD_PRIO); \
35 : : } while (false)
36 : : #else
37 : : #define Z_VALID_PRIO(prio, entry_point) ((prio) == -1)
38 : : #define Z_ASSERT_VALID_PRIO(prio, entry_point) __ASSERT((prio) == -1, "")
39 : : #endif /* CONFIG_MULTITHREADING */
40 : :
41 : : #if (CONFIG_MP_MAX_NUM_CPUS == 1)
42 : : #define LOCK_SCHED_SPINLOCK
43 : : #else
44 : : #define LOCK_SCHED_SPINLOCK K_SPINLOCK(&_sched_spinlock)
45 : : #endif
46 : :
47 : : #ifdef __cplusplus
48 : : extern "C" {
49 : : #endif
50 : :
51 : : extern struct k_spinlock _sched_spinlock;
52 : :
53 : : extern struct k_thread _thread_dummy;
54 : :
55 : : void z_sched_init(void);
56 : : void z_unpend_thread_no_timeout(struct k_thread *thread);
57 : : struct k_thread *z_unpend1_no_timeout(_wait_q_t *wait_q);
58 : : int z_pend_curr(struct k_spinlock *lock, k_spinlock_key_t key,
59 : : _wait_q_t *wait_q, k_timeout_t timeout);
60 : : void z_pend_thread(struct k_thread *thread, _wait_q_t *wait_q,
61 : : k_timeout_t timeout);
62 : : void z_reschedule(struct k_spinlock *lock, k_spinlock_key_t key);
63 : : void z_reschedule_irqlock(uint32_t key);
64 : : void z_unpend_thread(struct k_thread *thread);
65 : : int z_unpend_all(_wait_q_t *wait_q);
66 : : bool z_thread_prio_set(struct k_thread *thread, int prio);
67 : : void *z_get_next_switch_handle(void *interrupted);
68 : :
69 : : /* Wrapper around z_get_next_switch_handle() for the benefit of
70 : : * non-SMP platforms that always pass a NULL interrupted handle.
71 : : * Exposes the (extremely) common early exit case in a context that
72 : : * can be (much) better optimized in surrounding C code. Takes a
73 : : * _current pointer that the outer scope surely already has to avoid
74 : : * having to refetch it across a lock.
75 : : *
76 : : * Mystic arts for cycle nerds, basically. Replace with
77 : : * z_get_next_switch_handle() if this becomes a maintenance hassle.
78 : : */
79 : : static ALWAYS_INLINE void *z_sched_next_handle(struct k_thread *curr)
80 : : {
81 : : #ifndef CONFIG_SMP
82 : : if (curr == _kernel.ready_q.cache) {
83 : : return NULL;
84 : : }
85 : : #endif
86 : : return z_get_next_switch_handle(NULL);
87 : : }
88 : :
89 : : void z_time_slice(void);
90 : : void z_reset_time_slice(struct k_thread *curr);
91 : : void z_sched_start(struct k_thread *thread);
92 : : void z_ready_thread(struct k_thread *thread);
93 : : void z_requeue_current(struct k_thread *curr);
94 : : struct k_thread *z_swap_next_thread(void);
95 : : void move_current_to_end_of_prio_q(void);
96 : :
97 : 0 : static inline void z_reschedule_unlocked(void)
98 : : {
99 : 0 : (void) z_reschedule_irqlock(arch_irq_lock());
100 : 0 : }
101 : :
102 : : static inline bool z_is_under_prio_ceiling(int prio)
103 : : {
104 : : return prio >= CONFIG_PRIORITY_CEILING;
105 : : }
106 : :
107 : : static inline int z_get_new_prio_with_ceiling(int prio)
108 : : {
109 : : return z_is_under_prio_ceiling(prio) ? prio : CONFIG_PRIORITY_CEILING;
110 : : }
111 : :
112 : : static inline bool z_is_prio1_higher_than_or_equal_to_prio2(int prio1, int prio2)
113 : : {
114 : : return prio1 <= prio2;
115 : : }
116 : :
117 : : static inline bool z_is_prio_higher_or_equal(int prio1, int prio2)
118 : : {
119 : : return z_is_prio1_higher_than_or_equal_to_prio2(prio1, prio2);
120 : : }
121 : :
122 : : static inline bool z_is_prio1_lower_than_or_equal_to_prio2(int prio1, int prio2)
123 : : {
124 : : return prio1 >= prio2;
125 : : }
126 : :
127 : 55 : static inline bool z_is_prio1_higher_than_prio2(int prio1, int prio2)
128 : : {
129 : 55 : return prio1 < prio2;
130 : : }
131 : :
132 : 55 : static inline bool z_is_prio_higher(int prio, int test_prio)
133 : : {
134 : 55 : return z_is_prio1_higher_than_prio2(prio, test_prio);
135 : : }
136 : :
137 : : static inline bool z_is_prio_lower_or_equal(int prio1, int prio2)
138 : : {
139 : : return z_is_prio1_lower_than_or_equal_to_prio2(prio1, prio2);
140 : : }
141 : :
142 : : static inline bool _is_valid_prio(int prio, k_thread_entry_t entry_point)
143 : : {
144 : : if ((prio == K_IDLE_PRIO) && z_is_idle_thread_entry(entry_point)) {
145 : : return true;
146 : : }
147 : :
148 : : if (!z_is_prio_higher_or_equal(prio,
149 : : K_LOWEST_APPLICATION_THREAD_PRIO)) {
150 : : return false;
151 : : }
152 : :
153 : : if (!z_is_prio_lower_or_equal(prio,
154 : : K_HIGHEST_APPLICATION_THREAD_PRIO)) {
155 : : return false;
156 : : }
157 : :
158 : : return true;
159 : : }
160 : :
161 : 10 : static ALWAYS_INLINE _wait_q_t *pended_on_thread(struct k_thread *thread)
162 : : {
163 [ - + ]: 10 : __ASSERT_NO_MSG(thread->base.pended_on);
164 : :
165 : 10 : return thread->base.pended_on;
166 : : }
167 : :
168 : :
169 : 10 : static inline void unpend_thread_no_timeout(struct k_thread *thread)
170 : : {
171 : 10 : _priq_wait_remove(&pended_on_thread(thread)->waitq, thread);
172 : 10 : z_mark_thread_as_not_pending(thread);
173 : 10 : thread->base.pended_on = NULL;
174 : 10 : }
175 : :
176 : : /*
177 : : * In a multiprocessor system, z_unpend_first_thread() must lock the scheduler
178 : : * spinlock _sched_spinlock. However, in a uniprocessor system, that is not
179 : : * necessary as the caller has already taken precautions (in the form of
180 : : * locking interrupts).
181 : : */
182 : 6 : static ALWAYS_INLINE struct k_thread *z_unpend_first_thread(_wait_q_t *wait_q)
183 : : {
184 : 6 : struct k_thread *thread = NULL;
185 : :
186 [ - + ]: 6 : __ASSERT_EVAL(, int key = arch_irq_lock(); arch_irq_unlock(key),
187 : : !arch_irq_unlocked(key), "");
188 : :
189 : : LOCK_SCHED_SPINLOCK {
190 : 6 : thread = _priq_wait_best(&wait_q->waitq);
191 [ + + ]: 6 : if (unlikely(thread != NULL)) {
192 : 4 : unpend_thread_no_timeout(thread);
193 : 4 : z_abort_thread_timeout(thread);
194 : : }
195 : : }
196 : :
197 : 6 : return thread;
198 : : }
199 : :
200 : : /*
201 : : * APIs for working with the Zephyr kernel scheduler. Intended for use in
202 : : * management of IPC objects, either in the core kernel or other IPC
203 : : * implemented by OS compatibility layers, providing basic wait/wake operations
204 : : * with spinlocks used for synchronization.
205 : : *
206 : : * These APIs are public and will be treated as contract, even if the
207 : : * underlying scheduler implementation changes.
208 : : */
209 : :
210 : : /**
211 : : * Wake up a thread pending on the provided wait queue
212 : : *
213 : : * Given a wait_q, wake up the highest priority thread on the queue. If the
214 : : * queue was empty just return false.
215 : : *
216 : : * Otherwise, do the following, in order, holding _sched_spinlock the entire
217 : : * time so that the thread state is guaranteed not to change:
218 : : * - Set the thread's swap return values to swap_retval and swap_data
219 : : * - un-pend and ready the thread, but do not invoke the scheduler.
220 : : *
221 : : * Repeated calls to this function until it returns false is a suitable
222 : : * way to wake all threads on the queue.
223 : : *
224 : : * It is up to the caller to implement locking such that the return value of
225 : : * this function (whether a thread was woken up or not) does not immediately
226 : : * become stale. Calls to wait and wake on the same wait_q object must have
227 : : * synchronization. Calling this without holding any spinlock is a sign that
228 : : * this API is not being used properly.
229 : : *
230 : : * @param wait_q Wait queue to wake up the highest prio thread
231 : : * @param swap_retval Swap return value for woken thread
232 : : * @param swap_data Data return value to supplement swap_retval. May be NULL.
233 : : * @retval true If a thread was woken up
234 : : * @retval false If the wait_q was empty
235 : : */
236 : : bool z_sched_wake(_wait_q_t *wait_q, int swap_retval, void *swap_data);
237 : :
238 : : /**
239 : : * Wakes the specified thread.
240 : : *
241 : : * Given a specific thread, wake it up. This routine assumes that the given
242 : : * thread is not on the timeout queue.
243 : : *
244 : : * @warning Caller must hold _sched_spinlock when calling this function!
245 : : *
246 : : * @param thread Given thread to wake up.
247 : : *
248 : : */
249 : : void z_sched_wake_thread_locked(struct k_thread *thread);
250 : :
251 : : /**
252 : : * Wake up all threads pending on the provided wait queue
253 : : *
254 : : * Convenience function to invoke z_sched_wake() on all threads in the queue
255 : : * until there are no more to wake up.
256 : : *
257 : : * @param wait_q Wait queue to wake up the highest prio thread
258 : : * @param swap_retval Swap return value for woken thread
259 : : * @param swap_data Data return value to supplement swap_retval. May be NULL.
260 : : * @retval true If any threads were woken up
261 : : * @retval false If the wait_q was empty
262 : : */
263 : : static inline bool z_sched_wake_all(_wait_q_t *wait_q, int swap_retval,
264 : : void *swap_data)
265 : : {
266 : : bool woken = false;
267 : :
268 : : while (z_sched_wake(wait_q, swap_retval, swap_data)) {
269 : : woken = true;
270 : : }
271 : :
272 : : /* True if we woke at least one thread up */
273 : : return woken;
274 : : }
275 : :
276 : : /**
277 : : * Atomically put the current thread to sleep on a wait queue, with timeout
278 : : *
279 : : * The thread will be added to the provided waitqueue. The lock, which should
280 : : * be held by the caller with the provided key, will be released once this is
281 : : * completely done and we have swapped out.
282 : : *
283 : : * The return value and data pointer is set by whoever woke us up via
284 : : * z_sched_wake.
285 : : *
286 : : * @param lock Address of spinlock to release when we swap out
287 : : * @param key Key to the provided spinlock when it was locked
288 : : * @param wait_q Wait queue to go to sleep on
289 : : * @param timeout Waiting period to be woken up, or K_FOREVER to wait
290 : : * indefinitely.
291 : : * @param data Storage location for data pointer set when thread was woken up.
292 : : * May be NULL if not used.
293 : : * @retval Return value set by whatever woke us up, or -EAGAIN if the timeout
294 : : * expired without being woken up.
295 : : */
296 : : int z_sched_wait(struct k_spinlock *lock, k_spinlock_key_t key,
297 : : _wait_q_t *wait_q, k_timeout_t timeout, void **data);
298 : :
299 : : /**
300 : : * Callback function called during queue walk by @ref z_sched_waitq_walk
301 : : * for each thread in the wait queue (if not ended earlier)
302 : : *
303 : : * @param thread Thread info data structure
304 : : * @param data `data` parameter of z_sched_waitq_walk()
305 : : * @return non-zero to end wait queue walk immediately, 0 to continue
306 : : */
307 : : typedef int (*_waitq_walk_cb_t)(struct k_thread *thread, void *data);
308 : :
309 : : /**
310 : : * Callback function called after queue walk by @ref z_sched_waitq_walk
311 : : *
312 : : * @param status Wait queue walk status
313 : : * (same value that will be returned to caller of z_sched_waitq_walk())
314 : : * @param data `data` parameter of z_sched_waitq_walk()
315 : : */
316 : : typedef void (*_waitq_post_walk_cb_t)(int status, void *data);
317 : :
318 : : /**
319 : : * @brief Walks the wait queue invoking a callback on each waiting thread
320 : : *
321 : : * This function walks the wait queue invoking the `walk_func` callback function
322 : : * on each waiting thread (except if stopped earlier by a non-zero return value),
323 : : * followed by a single call of `post_func`, all while holding `_sched_spinlock`.
324 : : * This can be useful for routines that need to operate on multiple waiting threads.
325 : : *
326 : : * CAUTION! As a wait queue is of indeterminate length, the scheduler will be
327 : : * locked for an indeterminate amount of time. This may impact system performance.
328 : : * As such, care must be taken when using this function and in the callbacks.
329 : : *
330 : : * @warning @p walk_func may safely remove the thread received as argument from
331 : : * the wait queue only when `CONFIG_WAITQ_SCALABLE=n`.
332 : : *
333 : : * @param wait_q Identifies the wait queue to walk
334 : : * @param walk_func Callback to invoke for each waiting thread
335 : : * @param post_func Callback to invoke after queue walk (optional)
336 : : * @param data Custom data passed to the callbacks
337 : : *
338 : : * @return non-zero if walk is terminated by the callback; otherwise 0
339 : : */
340 : : int z_sched_waitq_walk(_wait_q_t *wait_q, _waitq_walk_cb_t walk_func,
341 : : _waitq_post_walk_cb_t post_func, void *data);
342 : :
343 : : /** @brief Halt thread cycle usage accounting.
344 : : *
345 : : * Halts the accumulation of thread cycle usage and adds the current
346 : : * total to the thread's counter. Called on context switch.
347 : : *
348 : : * Note that this function is idempotent. The core kernel code calls
349 : : * it at the end of interrupt handlers (because that is where we have
350 : : * a portable hook) where we are context switching, which will include
351 : : * any cycles spent in the ISR in the per-thread accounting. But
352 : : * architecture code can also call it earlier out of interrupt entry
353 : : * to improve measurement fidelity.
354 : : *
355 : : * This function assumes local interrupts are masked (so that the
356 : : * current CPU pointer and current thread are safe to modify), but
357 : : * requires no other synchronization. Architecture layers don't need
358 : : * to do anything more.
359 : : */
360 : : void z_sched_usage_stop(void);
361 : :
362 : : void z_sched_usage_start(struct k_thread *thread);
363 : :
364 : : /**
365 : : * @brief Retrieves CPU cycle usage data for specified core
366 : : */
367 : : void z_sched_cpu_usage(uint8_t core_id, struct k_thread_runtime_stats *stats);
368 : :
369 : : /**
370 : : * @brief Retrieves thread cycle usage data for specified thread
371 : : */
372 : : void z_sched_thread_usage(struct k_thread *thread,
373 : : struct k_thread_runtime_stats *stats);
374 : :
375 : : static inline void z_sched_usage_switch(struct k_thread *thread)
376 : : {
377 : : ARG_UNUSED(thread);
378 : : #ifdef CONFIG_SCHED_THREAD_USAGE
379 : : z_sched_usage_stop();
380 : : z_sched_usage_start(thread);
381 : : #endif /* CONFIG_SCHED_THREAD_USAGE */
382 : : }
383 : :
384 : : #ifdef __cplusplus
385 : : }
386 : : #endif
387 : :
388 : : #endif /* ZEPHYR_KERNEL_INCLUDE_KSCHED_H_ */
|
