개념 설명 전체 · v6.18.37 / kernel/sched/core.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel CPU scheduler code 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 * Copyright (C) 1998-2024 Ingo Molnar, Red Hat 9 */ 10 #define INSTANTIATE_EXPORTED_MIGRATE_DISABLE 11 #include <linux/sched.h> 12 #include <linux/highmem.h> 13 #include <linux/hrtimer_api.h> 14 #include <linux/ktime_api.h> 15 #include <linux/sched/signal.h> 16 #include <linux/syscalls_api.h> 17 #include <linux/debug_locks.h> 18 #include <linux/prefetch.h> 19 #include <linux/capability.h> 20 #include <linux/pgtable_api.h> 21 #include <linux/wait_bit.h> 22 #include <linux/jiffies.h> 23 #include <linux/spinlock_api.h> 24 #include <linux/cpumask_api.h> 25 #include <linux/lockdep_api.h> 26 #include <linux/hardirq.h> 27 #include <linux/softirq.h> 28 #include <linux/refcount_api.h> 29 #include <linux/topology.h> 30 #include <linux/sched/clock.h> 31 #include <linux/sched/cond_resched.h> 32 #include <linux/sched/cputime.h> 33 #include <linux/sched/debug.h> 34 #include <linux/sched/hotplug.h> 35 #include <linux/sched/init.h> 36 #include <linux/sched/isolation.h> 37 #include <linux/sched/loadavg.h> 38 #include <linux/sched/mm.h> 39 #include <linux/sched/nohz.h> 40 #include <linux/sched/rseq_api.h> 41 #include <linux/sched/rt.h> 42 43 #include <linux/blkdev.h> 44 #include <linux/context_tracking.h> 45 #include <linux/cpuset.h> 46 #include <linux/delayacct.h> 47 #include <linux/init_task.h> 48 #include <linux/interrupt.h> 49 #include <linux/ioprio.h> 50 #include <linux/kallsyms.h> 51 #include <linux/kcov.h> 52 #include <linux/kprobes.h> 53 #include <linux/llist_api.h> 54 #include <linux/mmu_context.h> 55 #include <linux/mmzone.h> 56 #include <linux/mutex_api.h> 57 #include <linux/nmi.h> 58 #include <linux/nospec.h> 59 #include <linux/perf_event_api.h> 60 #include <linux/profile.h> 61 #include <linux/psi.h> 62 #include <linux/rcuwait_api.h> 63 #include <linux/rseq.h> 64 #include <linux/sched/wake_q.h> 65 #include <linux/scs.h> 66 #include <linux/slab.h> 67 #include <linux/syscalls.h> 68 #include <linux/vtime.h> 69 #include <linux/wait_api.h> 70 #include <linux/workqueue_api.h> 71 #include <linux/livepatch_sched.h> 72 73 #ifdef CONFIG_PREEMPT_DYNAMIC 74 # ifdef CONFIG_GENERIC_IRQ_ENTRY 75 # include <linux/irq-entry-common.h> 76 # endif 77 #endif 78 79 #include <uapi/linux/sched/types.h> 80 81 #include <asm/irq_regs.h> 82 #include <asm/switch_to.h> 83 #include <asm/tlb.h> 84 85 #define CREATE_TRACE_POINTS 86 #include <linux/sched/rseq_api.h> 87 #include <trace/events/sched.h> 88 #include <trace/events/ipi.h> 89 #undef CREATE_TRACE_POINTS 90 91 #include "sched.h" 92 #include "stats.h" 93 94 #include "autogroup.h" 95 #include "pelt.h" 96 #include "smp.h" 97 98 #include "../workqueue_internal.h" 99 #include "../../io_uring/io-wq.h" 100 #include "../smpboot.h" 101 #include "../locking/mutex.h" 102 103 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); 104 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); 105 106 /* 107 * Export tracepoints that act as a bare tracehook (ie: have no trace event 108 * associated with them) to allow external modules to probe them. 109 */ 110 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 111 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 112 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 113 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 114 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 115 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_hw_tp); 116 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 117 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 118 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 119 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 120 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 121 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); 122 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); 123 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); 124 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); 125 126 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 127 DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); 128 129 #ifdef CONFIG_SCHED_PROXY_EXEC 130 DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec); 131 static int __init setup_proxy_exec(char *str) 132 { 133 bool proxy_enable = true; 134 135 if (*str && kstrtobool(str + 1, &proxy_enable)) { 136 pr_warn("Unable to parse sched_proxy_exec=\n"); 137 return 0; 138 } 139 140 if (proxy_enable) { 141 pr_info("sched_proxy_exec enabled via boot arg\n"); 142 static_branch_enable(&__sched_proxy_exec); 143 } else { 144 pr_info("sched_proxy_exec disabled via boot arg\n"); 145 static_branch_disable(&__sched_proxy_exec); 146 } 147 return 1; 148 } 149 #else 150 static int __init setup_proxy_exec(char *str) 151 { 152 pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n"); 153 return 0; 154 } 155 #endif 156 __setup("sched_proxy_exec", setup_proxy_exec); 157 158 /* 159 * Debugging: various feature bits 160 * 161 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 162 * sysctl_sched_features, defined in sched.h, to allow constants propagation 163 * at compile time and compiler optimization based on features default. 164 */ 165 #define SCHED_FEAT(name, enabled) \ 166 (1UL << __SCHED_FEAT_##name) * enabled | 167 __read_mostly unsigned int sysctl_sched_features = 168 #include "features.h" 169 0; 170 #undef SCHED_FEAT 171 172 /* 173 * Print a warning if need_resched is set for the given duration (if 174 * LATENCY_WARN is enabled). 175 * 176 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 177 * per boot. 178 */ 179 __read_mostly int sysctl_resched_latency_warn_ms = 100; 180 __read_mostly int sysctl_resched_latency_warn_once = 1; 181 182 /* 183 * Number of tasks to iterate in a single balance run. 184 * Limited because this is done with IRQs disabled. 185 */ 186 __read_mostly unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; 187 188 __read_mostly int scheduler_running; 189 190 #ifdef CONFIG_SCHED_CORE 191 192 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 193 194 /* kernel prio, less is more */ 195 static inline int __task_prio(const struct task_struct *p) 196 { 197 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 198 return -2; 199 200 if (p->dl_server) 201 return -1; /* deadline */ 202 203 if (rt_or_dl_prio(p->prio)) 204 return p->prio; /* [-1, 99] */ 205 206 if (p->sched_class == &idle_sched_class) 207 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 208 209 if (task_on_scx(p)) 210 return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ 211 212 return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ 213 } 214 215 /* 216 * l(a,b) 217 * le(a,b) := !l(b,a) 218 * g(a,b) := l(b,a) 219 * ge(a,b) := !l(a,b) 220 */ 221 222 /* real prio, less is less */ 223 static inline bool prio_less(const struct task_struct *a, 224 const struct task_struct *b, bool in_fi) 225 { 226 227 int pa = __task_prio(a), pb = __task_prio(b); 228 229 if (-pa < -pb) 230 return true; 231 232 if (-pb < -pa) 233 return false; 234 235 if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */ 236 const struct sched_dl_entity *a_dl, *b_dl; 237 238 a_dl = &a->dl; 239 /* 240 * Since,'a' and 'b' can be CFS tasks served by DL server, 241 * __task_prio() can return -1 (for DL) even for those. In that 242 * case, get to the dl_server's DL entity. 243 */ 244 if (a->dl_server) 245 a_dl = a->dl_server; 246 247 b_dl = &b->dl; 248 if (b->dl_server) 249 b_dl = b->dl_server; 250 251 return !dl_time_before(a_dl->deadline, b_dl->deadline); 252 } 253 254 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 255 return cfs_prio_less(a, b, in_fi); 256 257 #ifdef CONFIG_SCHED_CLASS_EXT 258 if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ 259 return scx_prio_less(a, b, in_fi); 260 #endif 261 262 return false; 263 } 264 265 static inline bool __sched_core_less(const struct task_struct *a, 266 const struct task_struct *b) 267 { 268 if (a->core_cookie < b->core_cookie) 269 return true; 270 271 if (a->core_cookie > b->core_cookie) 272 return false; 273 274 /* flip prio, so high prio is leftmost */ 275 if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) 276 return true; 277 278 return false; 279 } 280 281 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 282 283 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 284 { 285 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 286 } 287 288 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 289 { 290 const struct task_struct *p = __node_2_sc(node); 291 unsigned long cookie = (unsigned long)key; 292 293 if (cookie < p->core_cookie) 294 return -1; 295 296 if (cookie > p->core_cookie) 297 return 1; 298 299 return 0; 300 } 301 302 void sched_core_enqueue(struct rq *rq, struct task_struct *p) 303 { 304 if (p->se.sched_delayed) 305 return; 306 307 rq->core->core_task_seq++; 308 309 if (!p->core_cookie) 310 return; 311 312 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 313 } 314 315 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) 316 { 317 if (p->se.sched_delayed) 318 return; 319 320 rq->core->core_task_seq++; 321 322 if (sched_core_enqueued(p)) { 323 rb_erase(&p->core_node, &rq->core_tree); 324 RB_CLEAR_NODE(&p->core_node); 325 } 326 327 /* 328 * Migrating the last task off the cpu, with the cpu in forced idle 329 * state. Reschedule to create an accounting edge for forced idle, 330 * and re-examine whether the core is still in forced idle state. 331 */ 332 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && 333 rq->core->core_forceidle_count && rq->curr == rq->idle) 334 resched_curr(rq); 335 } 336 337 static int sched_task_is_throttled(struct task_struct *p, int cpu) 338 { 339 if (p->sched_class->task_is_throttled) 340 return p->sched_class->task_is_throttled(p, cpu); 341 342 return 0; 343 } 344 345 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 346 { 347 struct rb_node *node = &p->core_node; 348 int cpu = task_cpu(p); 349 350 do { 351 node = rb_next(node); 352 if (!node) 353 return NULL; 354 355 p = __node_2_sc(node); 356 if (p->core_cookie != cookie) 357 return NULL; 358 359 } while (sched_task_is_throttled(p, cpu)); 360 361 return p; 362 } 363 364 /* 365 * Find left-most (aka, highest priority) and unthrottled task matching @cookie. 366 * If no suitable task is found, NULL will be returned. 367 */ 368 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 369 { 370 struct task_struct *p; 371 struct rb_node *node; 372 373 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 374 if (!node) 375 return NULL; 376 377 p = __node_2_sc(node); 378 if (!sched_task_is_throttled(p, rq->cpu)) 379 return p; 380 381 return sched_core_next(p, cookie); 382 } 383 384 /* 385 * Magic required such that: 386 * 387 * raw_spin_rq_lock(rq); 388 * ... 389 * raw_spin_rq_unlock(rq); 390 * 391 * ends up locking and unlocking the _same_ lock, and all CPUs 392 * always agree on what rq has what lock. 393 * 394 * XXX entirely possible to selectively enable cores, don't bother for now. 395 */ 396 397 static DEFINE_MUTEX(sched_core_mutex); 398 static atomic_t sched_core_count; 399 static struct cpumask sched_core_mask; 400 401 static void sched_core_lock(int cpu, unsigned long *flags) 402 { 403 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 404 int t, i = 0; 405 406 local_irq_save(*flags); 407 for_each_cpu(t, smt_mask) 408 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 409 } 410 411 static void sched_core_unlock(int cpu, unsigned long *flags) 412 { 413 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 414 int t; 415 416 for_each_cpu(t, smt_mask) 417 raw_spin_unlock(&cpu_rq(t)->__lock); 418 local_irq_restore(*flags); 419 } 420 421 static void __sched_core_flip(bool enabled) 422 { 423 unsigned long flags; 424 int cpu, t; 425 426 cpus_read_lock(); 427 428 /* 429 * Toggle the online cores, one by one. 430 */ 431 cpumask_copy(&sched_core_mask, cpu_online_mask); 432 for_each_cpu(cpu, &sched_core_mask) { 433 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 434 435 sched_core_lock(cpu, &flags); 436 437 for_each_cpu(t, smt_mask) 438 cpu_rq(t)->core_enabled = enabled; 439 440 cpu_rq(cpu)->core->core_forceidle_start = 0; 441 442 sched_core_unlock(cpu, &flags); 443 444 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 445 } 446 447 /* 448 * Toggle the offline CPUs. 449 */ 450 for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 451 cpu_rq(cpu)->core_enabled = enabled; 452 453 cpus_read_unlock(); 454 } 455 456 static void sched_core_assert_empty(void) 457 { 458 int cpu; 459 460 for_each_possible_cpu(cpu) 461 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 462 } 463 464 static void __sched_core_enable(void) 465 { 466 static_branch_enable(&__sched_core_enabled); 467 /* 468 * Ensure all previous instances of raw_spin_rq_*lock() have finished 469 * and future ones will observe !sched_core_disabled(). 470 */ 471 synchronize_rcu(); 472 __sched_core_flip(true); 473 sched_core_assert_empty(); 474 } 475 476 static void __sched_core_disable(void) 477 { 478 sched_core_assert_empty(); 479 __sched_core_flip(false); 480 static_branch_disable(&__sched_core_enabled); 481 } 482 483 void sched_core_get(void) 484 { 485 if (atomic_inc_not_zero(&sched_core_count)) 486 return; 487 488 mutex_lock(&sched_core_mutex); 489 if (!atomic_read(&sched_core_count)) 490 __sched_core_enable(); 491 492 smp_mb__before_atomic(); 493 atomic_inc(&sched_core_count); 494 mutex_unlock(&sched_core_mutex); 495 } 496 497 static void __sched_core_put(struct work_struct *work) 498 { 499 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 500 __sched_core_disable(); 501 mutex_unlock(&sched_core_mutex); 502 } 503 } 504 505 void sched_core_put(void) 506 { 507 static DECLARE_WORK(_work, __sched_core_put); 508 509 /* 510 * "There can be only one" 511 * 512 * Either this is the last one, or we don't actually need to do any 513 * 'work'. If it is the last *again*, we rely on 514 * WORK_STRUCT_PENDING_BIT. 515 */ 516 if (!atomic_add_unless(&sched_core_count, -1, 1)) 517 schedule_work(&_work); 518 } 519 520 #else /* !CONFIG_SCHED_CORE: */ 521 522 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 523 static inline void 524 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } 525 526 #endif /* !CONFIG_SCHED_CORE */ 527 528 /* need a wrapper since we may need to trace from modules */ 529 EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp); 530 531 /* Call via the helper macro trace_set_current_state. */ 532 void __trace_set_current_state(int state_value) 533 { 534 trace_sched_set_state_tp(current, state_value); 535 } 536 EXPORT_SYMBOL(__trace_set_current_state); 537 538 /* 539 * Serialization rules: 540 * 541 * Lock order: 542 * 543 * p->pi_lock 544 * rq->lock 545 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 546 * 547 * rq1->lock 548 * rq2->lock where: rq1 < rq2 549 * 550 * Regular state: 551 * 552 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 553 * local CPU's rq->lock, it optionally removes the task from the runqueue and 554 * always looks at the local rq data structures to find the most eligible task 555 * to run next. 556 * 557 * Task enqueue is also under rq->lock, possibly taken from another CPU. 558 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 559 * the local CPU to avoid bouncing the runqueue state around [ see 560 * ttwu_queue_wakelist() ] 561 * 562 * Task wakeup, specifically wakeups that involve migration, are horribly 563 * complicated to avoid having to take two rq->locks. 564 * 565 * Special state: 566 * 567 * System-calls and anything external will use task_rq_lock() which acquires 568 * both p->pi_lock and rq->lock. As a consequence the state they change is 569 * stable while holding either lock: 570 * 571 * - sched_setaffinity()/ 572 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 573 * - set_user_nice(): p->se.load, p->*prio 574 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 575 * p->se.load, p->rt_priority, 576 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 577 * - sched_setnuma(): p->numa_preferred_nid 578 * - sched_move_task(): p->sched_task_group 579 * - uclamp_update_active() p->uclamp* 580 * 581 * p->state <- TASK_*: 582 * 583 * is changed locklessly using set_current_state(), __set_current_state() or 584 * set_special_state(), see their respective comments, or by 585 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 586 * concurrent self. 587 * 588 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 589 * 590 * is set by activate_task() and cleared by deactivate_task(), under 591 * rq->lock. Non-zero indicates the task is runnable, the special 592 * ON_RQ_MIGRATING state is used for migration without holding both 593 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 594 * 595 * Additionally it is possible to be ->on_rq but still be considered not 596 * runnable when p->se.sched_delayed is true. These tasks are on the runqueue 597 * but will be dequeued as soon as they get picked again. See the 598 * task_is_runnable() helper. 599 * 600 * p->on_cpu <- { 0, 1 }: 601 * 602 * is set by prepare_task() and cleared by finish_task() such that it will be 603 * set before p is scheduled-in and cleared after p is scheduled-out, both 604 * under rq->lock. Non-zero indicates the task is running on its CPU. 605 * 606 * [ The astute reader will observe that it is possible for two tasks on one 607 * CPU to have ->on_cpu = 1 at the same time. ] 608 * 609 * task_cpu(p): is changed by set_task_cpu(), the rules are: 610 * 611 * - Don't call set_task_cpu() on a blocked task: 612 * 613 * We don't care what CPU we're not running on, this simplifies hotplug, 614 * the CPU assignment of blocked tasks isn't required to be valid. 615 * 616 * - for try_to_wake_up(), called under p->pi_lock: 617 * 618 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 619 * 620 * - for migration called under rq->lock: 621 * [ see task_on_rq_migrating() in task_rq_lock() ] 622 * 623 * o move_queued_task() 624 * o detach_task() 625 * 626 * - for migration called under double_rq_lock(): 627 * 628 * o __migrate_swap_task() 629 * o push_rt_task() / pull_rt_task() 630 * o push_dl_task() / pull_dl_task() 631 * o dl_task_offline_migration() 632 * 633 */ 634 635 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 636 { 637 raw_spinlock_t *lock; 638 639 /* Matches synchronize_rcu() in __sched_core_enable() */ 640 preempt_disable(); 641 if (sched_core_disabled()) { 642 raw_spin_lock_nested(&rq->__lock, subclass); 643 /* preempt_count *MUST* be > 1 */ 644 preempt_enable_no_resched(); 645 return; 646 } 647 648 for (;;) { 649 lock = __rq_lockp(rq); 650 raw_spin_lock_nested(lock, subclass); 651 if (likely(lock == __rq_lockp(rq))) { 652 /* preempt_count *MUST* be > 1 */ 653 preempt_enable_no_resched(); 654 return; 655 } 656 raw_spin_unlock(lock); 657 } 658 } 659 660 bool raw_spin_rq_trylock(struct rq *rq) 661 { 662 raw_spinlock_t *lock; 663 bool ret; 664 665 /* Matches synchronize_rcu() in __sched_core_enable() */ 666 preempt_disable(); 667 if (sched_core_disabled()) { 668 ret = raw_spin_trylock(&rq->__lock); 669 preempt_enable(); 670 return ret; 671 } 672 673 for (;;) { 674 lock = __rq_lockp(rq); 675 ret = raw_spin_trylock(lock); 676 if (!ret || (likely(lock == __rq_lockp(rq)))) { 677 preempt_enable(); 678 return ret; 679 } 680 raw_spin_unlock(lock); 681 } 682 } 683 684 void raw_spin_rq_unlock(struct rq *rq) 685 { 686 raw_spin_unlock(rq_lockp(rq)); 687 } 688 689 /* 690 * double_rq_lock - safely lock two runqueues 691 */ 692 void double_rq_lock(struct rq *rq1, struct rq *rq2) 693 { 694 lockdep_assert_irqs_disabled(); 695 696 if (rq_order_less(rq2, rq1)) 697 swap(rq1, rq2); 698 699 raw_spin_rq_lock(rq1); 700 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 701 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 702 703 double_rq_clock_clear_update(rq1, rq2); 704 } 705 706 /* 707 * __task_rq_lock - lock the rq @p resides on. 708 */ 709 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 710 __acquires(rq->lock) 711 { 712 struct rq *rq; 713 714 lockdep_assert_held(&p->pi_lock); 715 716 for (;;) { 717 rq = task_rq(p); 718 raw_spin_rq_lock(rq); 719 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 720 rq_pin_lock(rq, rf); 721 return rq; 722 } 723 raw_spin_rq_unlock(rq); 724 725 while (unlikely(task_on_rq_migrating(p))) 726 cpu_relax(); 727 } 728 } 729 730 /* 731 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 732 */ 733 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 734 __acquires(p->pi_lock) 735 __acquires(rq->lock) 736 { 737 struct rq *rq; 738 739 for (;;) { 740 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 741 rq = task_rq(p); 742 raw_spin_rq_lock(rq); 743 /* 744 * move_queued_task() task_rq_lock() 745 * 746 * ACQUIRE (rq->lock) 747 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 748 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 749 * [S] ->cpu = new_cpu [L] task_rq() 750 * [L] ->on_rq 751 * RELEASE (rq->lock) 752 * 753 * If we observe the old CPU in task_rq_lock(), the acquire of 754 * the old rq->lock will fully serialize against the stores. 755 * 756 * If we observe the new CPU in task_rq_lock(), the address 757 * dependency headed by '[L] rq = task_rq()' and the acquire 758 * will pair with the WMB to ensure we then also see migrating. 759 */ 760 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 761 rq_pin_lock(rq, rf); 762 return rq; 763 } 764 raw_spin_rq_unlock(rq); 765 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 766 767 while (unlikely(task_on_rq_migrating(p))) 768 cpu_relax(); 769 } 770 } 771 772 /* 773 * RQ-clock updating methods: 774 */ 775 776 static void update_rq_clock_task(struct rq *rq, s64 delta) 777 { 778 /* 779 * In theory, the compile should just see 0 here, and optimize out the call 780 * to sched_rt_avg_update. But I don't trust it... 781 */ 782 s64 __maybe_unused steal = 0, irq_delta = 0; 783 784 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 785 if (irqtime_enabled()) { 786 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 787 788 /* 789 * Since irq_time is only updated on {soft,}irq_exit, we might run into 790 * this case when a previous update_rq_clock() happened inside a 791 * {soft,}IRQ region. 792 * 793 * When this happens, we stop ->clock_task and only update the 794 * prev_irq_time stamp to account for the part that fit, so that a next 795 * update will consume the rest. This ensures ->clock_task is 796 * monotonic. 797 * 798 * It does however cause some slight miss-attribution of {soft,}IRQ 799 * time, a more accurate solution would be to update the irq_time using 800 * the current rq->clock timestamp, except that would require using 801 * atomic ops. 802 */ 803 if (irq_delta > delta) 804 irq_delta = delta; 805 806 rq->prev_irq_time += irq_delta; 807 delta -= irq_delta; 808 delayacct_irq(rq->curr, irq_delta); 809 } 810 #endif 811 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 812 if (static_key_false((¶virt_steal_rq_enabled))) { 813 u64 prev_steal; 814 815 steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); 816 steal -= rq->prev_steal_time_rq; 817 818 if (unlikely(steal > delta)) 819 steal = delta; 820 821 rq->prev_steal_time_rq = prev_steal; 822 delta -= steal; 823 } 824 #endif 825 826 rq->clock_task += delta; 827 828 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 829 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 830 update_irq_load_avg(rq, irq_delta + steal); 831 #endif 832 update_rq_clock_pelt(rq, delta); 833 } 834 835 void update_rq_clock(struct rq *rq) 836 { 837 s64 delta; 838 u64 clock; 839 840 lockdep_assert_rq_held(rq); 841 842 if (rq->clock_update_flags & RQCF_ACT_SKIP) 843 return; 844 845 if (sched_feat(WARN_DOUBLE_CLOCK)) 846 WARN_ON_ONCE(rq->clock_update_flags & RQCF_UPDATED); 847 rq->clock_update_flags |= RQCF_UPDATED; 848 849 clock = sched_clock_cpu(cpu_of(rq)); 850 scx_rq_clock_update(rq, clock); 851 852 delta = clock - rq->clock; 853 if (delta < 0) 854 return; 855 rq->clock += delta; 856 857 update_rq_clock_task(rq, delta); 858 } 859 860 #ifdef CONFIG_SCHED_HRTICK 861 /* 862 * Use HR-timers to deliver accurate preemption points. 863 */ 864 865 static void hrtick_clear(struct rq *rq) 866 { 867 if (hrtimer_active(&rq->hrtick_timer)) 868 hrtimer_cancel(&rq->hrtick_timer); 869 } 870 871 /* 872 * High-resolution timer tick. 873 * Runs from hardirq context with interrupts disabled. 874 */ 875 static enum hrtimer_restart hrtick(struct hrtimer *timer) 876 { 877 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 878 struct rq_flags rf; 879 880 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 881 882 rq_lock(rq, &rf); 883 update_rq_clock(rq); 884 rq->donor->sched_class->task_tick(rq, rq->curr, 1); 885 rq_unlock(rq, &rf); 886 887 return HRTIMER_NORESTART; 888 } 889 890 static void __hrtick_restart(struct rq *rq) 891 { 892 struct hrtimer *timer = &rq->hrtick_timer; 893 ktime_t time = rq->hrtick_time; 894 895 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 896 } 897 898 /* 899 * called from hardirq (IPI) context 900 */ 901 static void __hrtick_start(void *arg) 902 { 903 struct rq *rq = arg; 904 struct rq_flags rf; 905 906 rq_lock(rq, &rf); 907 __hrtick_restart(rq); 908 rq_unlock(rq, &rf); 909 } 910 911 /* 912 * Called to set the hrtick timer state. 913 * 914 * called with rq->lock held and IRQs disabled 915 */ 916 void hrtick_start(struct rq *rq, u64 delay) 917 { 918 struct hrtimer *timer = &rq->hrtick_timer; 919 s64 delta; 920 921 /* 922 * Don't schedule slices shorter than 10000ns, that just 923 * doesn't make sense and can cause timer DoS. 924 */ 925 delta = max_t(s64, delay, 10000LL); 926 rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta); 927 928 if (rq == this_rq()) 929 __hrtick_restart(rq); 930 else 931 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 932 } 933 934 static void hrtick_rq_init(struct rq *rq) 935 { 936 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 937 hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 938 } 939 #else /* !CONFIG_SCHED_HRTICK: */ 940 static inline void hrtick_clear(struct rq *rq) 941 { 942 } 943 944 static inline void hrtick_rq_init(struct rq *rq) 945 { 946 } 947 #endif /* !CONFIG_SCHED_HRTICK */ 948 949 /* 950 * try_cmpxchg based fetch_or() macro so it works for different integer types: 951 */ 952 #define fetch_or(ptr, mask) \ 953 ({ \ 954 typeof(ptr) _ptr = (ptr); \ 955 typeof(mask) _mask = (mask); \ 956 typeof(*_ptr) _val = *_ptr; \ 957 \ 958 do { \ 959 } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ 960 _val; \ 961 }) 962 963 #ifdef TIF_POLLING_NRFLAG 964 /* 965 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 966 * this avoids any races wrt polling state changes and thereby avoids 967 * spurious IPIs. 968 */ 969 static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif) 970 { 971 return !(fetch_or(&ti->flags, 1 << tif) & _TIF_POLLING_NRFLAG); 972 } 973 974 /* 975 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 976 * 977 * If this returns true, then the idle task promises to call 978 * sched_ttwu_pending() and reschedule soon. 979 */ 980 static bool set_nr_if_polling(struct task_struct *p) 981 { 982 struct thread_info *ti = task_thread_info(p); 983 typeof(ti->flags) val = READ_ONCE(ti->flags); 984 985 do { 986 if (!(val & _TIF_POLLING_NRFLAG)) 987 return false; 988 if (val & _TIF_NEED_RESCHED) 989 return true; 990 } while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)); 991 992 return true; 993 } 994 995 #else 996 static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif) 997 { 998 set_ti_thread_flag(ti, tif); 999 return true; 1000 } 1001 1002 static inline bool set_nr_if_polling(struct task_struct *p) 1003 { 1004 return false; 1005 } 1006 #endif 1007 1008 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 1009 { 1010 struct wake_q_node *node = &task->wake_q; 1011 1012 /* 1013 * Atomically grab the task, if ->wake_q is !nil already it means 1014 * it's already queued (either by us or someone else) and will get the 1015 * wakeup due to that. 1016 * 1017 * In order to ensure that a pending wakeup will observe our pending 1018 * state, even in the failed case, an explicit smp_mb() must be used. 1019 */ 1020 smp_mb__before_atomic(); 1021 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 1022 return false; 1023 1024 /* 1025 * The head is context local, there can be no concurrency. 1026 */ 1027 *head->lastp = node; 1028 head->lastp = &node->next; 1029 return true; 1030 } 1031 1032 /** 1033 * wake_q_add() - queue a wakeup for 'later' waking. 1034 * @head: the wake_q_head to add @task to 1035 * @task: the task to queue for 'later' wakeup 1036 * 1037 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1038 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1039 * instantly. 1040 * 1041 * This function must be used as-if it were wake_up_process(); IOW the task 1042 * must be ready to be woken at this location. 1043 */ 1044 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 1045 { 1046 if (__wake_q_add(head, task)) 1047 get_task_struct(task); 1048 } 1049 1050 /** 1051 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 1052 * @head: the wake_q_head to add @task to 1053 * @task: the task to queue for 'later' wakeup 1054 * 1055 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1056 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1057 * instantly. 1058 * 1059 * This function must be used as-if it were wake_up_process(); IOW the task 1060 * must be ready to be woken at this location. 1061 * 1062 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 1063 * that already hold reference to @task can call the 'safe' version and trust 1064 * wake_q to do the right thing depending whether or not the @task is already 1065 * queued for wakeup. 1066 */ 1067 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 1068 { 1069 if (!__wake_q_add(head, task)) 1070 put_task_struct(task); 1071 } 1072 1073 void wake_up_q(struct wake_q_head *head) 1074 { 1075 struct wake_q_node *node = head->first; 1076 1077 while (node != WAKE_Q_TAIL) { 1078 struct task_struct *task; 1079 1080 task = container_of(node, struct task_struct, wake_q); 1081 node = node->next; 1082 /* pairs with cmpxchg_relaxed() in __wake_q_add() */ 1083 WRITE_ONCE(task->wake_q.next, NULL); 1084 /* Task can safely be re-inserted now. */ 1085 1086 /* 1087 * wake_up_process() executes a full barrier, which pairs with 1088 * the queueing in wake_q_add() so as not to miss wakeups. 1089 */ 1090 wake_up_process(task); 1091 put_task_struct(task); 1092 } 1093 } 1094 1095 /* 1096 * resched_curr - mark rq's current task 'to be rescheduled now'. 1097 * 1098 * On UP this means the setting of the need_resched flag, on SMP it 1099 * might also involve a cross-CPU call to trigger the scheduler on 1100 * the target CPU. 1101 */ 1102 static void __resched_curr(struct rq *rq, int tif) 1103 { 1104 struct task_struct *curr = rq->curr; 1105 struct thread_info *cti = task_thread_info(curr); 1106 int cpu; 1107 1108 lockdep_assert_rq_held(rq); 1109 1110 /* 1111 * Always immediately preempt the idle task; no point in delaying doing 1112 * actual work. 1113 */ 1114 if (is_idle_task(curr) && tif == TIF_NEED_RESCHED_LAZY) 1115 tif = TIF_NEED_RESCHED; 1116 1117 if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED)) 1118 return; 1119 1120 cpu = cpu_of(rq); 1121 1122 trace_sched_set_need_resched_tp(curr, cpu, tif); 1123 if (cpu == smp_processor_id()) { 1124 set_ti_thread_flag(cti, tif); 1125 if (tif == TIF_NEED_RESCHED) 1126 set_preempt_need_resched(); 1127 return; 1128 } 1129 1130 if (set_nr_and_not_polling(cti, tif)) { 1131 if (tif == TIF_NEED_RESCHED) 1132 smp_send_reschedule(cpu); 1133 } else { 1134 trace_sched_wake_idle_without_ipi(cpu); 1135 } 1136 } 1137 1138 void __trace_set_need_resched(struct task_struct *curr, int tif) 1139 { 1140 trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); 1141 } 1142 EXPORT_SYMBOL_GPL(__trace_set_need_resched); 1143 1144 void resched_curr(struct rq *rq) 1145 { 1146 __resched_curr(rq, TIF_NEED_RESCHED); 1147 } 1148 1149 #ifdef CONFIG_PREEMPT_DYNAMIC 1150 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy); 1151 static __always_inline bool dynamic_preempt_lazy(void) 1152 { 1153 return static_branch_unlikely(&sk_dynamic_preempt_lazy); 1154 } 1155 #else 1156 static __always_inline bool dynamic_preempt_lazy(void) 1157 { 1158 return IS_ENABLED(CONFIG_PREEMPT_LAZY); 1159 } 1160 #endif 1161 1162 static __always_inline int get_lazy_tif_bit(void) 1163 { 1164 if (dynamic_preempt_lazy()) 1165 return TIF_NEED_RESCHED_LAZY; 1166 1167 return TIF_NEED_RESCHED; 1168 } 1169 1170 void resched_curr_lazy(struct rq *rq) 1171 { 1172 __resched_curr(rq, get_lazy_tif_bit()); 1173 } 1174 1175 void resched_cpu(int cpu) 1176 { 1177 struct rq *rq = cpu_rq(cpu); 1178 unsigned long flags; 1179 1180 raw_spin_rq_lock_irqsave(rq, flags); 1181 if (cpu_online(cpu) || cpu == smp_processor_id()) 1182 resched_curr(rq); 1183 raw_spin_rq_unlock_irqrestore(rq, flags); 1184 } 1185 1186 #ifdef CONFIG_NO_HZ_COMMON 1187 /* 1188 * In the semi idle case, use the nearest busy CPU for migrating timers 1189 * from an idle CPU. This is good for power-savings. 1190 * 1191 * We don't do similar optimization for completely idle system, as 1192 * selecting an idle CPU will add more delays to the timers than intended 1193 * (as that CPU's timer base may not be up to date wrt jiffies etc). 1194 */ 1195 int get_nohz_timer_target(void) 1196 { 1197 int i, cpu = smp_processor_id(), default_cpu = -1; 1198 struct sched_domain *sd; 1199 const struct cpumask *hk_mask; 1200 1201 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) { 1202 if (!idle_cpu(cpu)) 1203 return cpu; 1204 default_cpu = cpu; 1205 } 1206 1207 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); 1208 1209 guard(rcu)(); 1210 1211 for_each_domain(cpu, sd) { 1212 for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { 1213 if (cpu == i) 1214 continue; 1215 1216 if (!idle_cpu(i)) 1217 return i; 1218 } 1219 } 1220 1221 if (default_cpu == -1) 1222 default_cpu = housekeeping_any_cpu(HK_TYPE_KERNEL_NOISE); 1223 1224 return default_cpu; 1225 } 1226 1227 /* 1228 * When add_timer_on() enqueues a timer into the timer wheel of an 1229 * idle CPU then this timer might expire before the next timer event 1230 * which is scheduled to wake up that CPU. In case of a completely 1231 * idle system the next event might even be infinite time into the 1232 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1233 * leaves the inner idle loop so the newly added timer is taken into 1234 * account when the CPU goes back to idle and evaluates the timer 1235 * wheel for the next timer event. 1236 */ 1237 static void wake_up_idle_cpu(int cpu) 1238 { 1239 struct rq *rq = cpu_rq(cpu); 1240 1241 if (cpu == smp_processor_id()) 1242 return; 1243 1244 /* 1245 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling 1246 * part of the idle loop. This forces an exit from the idle loop 1247 * and a round trip to schedule(). Now this could be optimized 1248 * because a simple new idle loop iteration is enough to 1249 * re-evaluate the next tick. Provided some re-ordering of tick 1250 * nohz functions that would need to follow TIF_NR_POLLING 1251 * clearing: 1252 * 1253 * - On most architectures, a simple fetch_or on ti::flags with a 1254 * "0" value would be enough to know if an IPI needs to be sent. 1255 * 1256 * - x86 needs to perform a last need_resched() check between 1257 * monitor and mwait which doesn't take timers into account. 1258 * There a dedicated TIF_TIMER flag would be required to 1259 * fetch_or here and be checked along with TIF_NEED_RESCHED 1260 * before mwait(). 1261 * 1262 * However, remote timer enqueue is not such a frequent event 1263 * and testing of the above solutions didn't appear to report 1264 * much benefits. 1265 */ 1266 if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED)) 1267 smp_send_reschedule(cpu); 1268 else 1269 trace_sched_wake_idle_without_ipi(cpu); 1270 } 1271 1272 static bool wake_up_full_nohz_cpu(int cpu) 1273 { 1274 /* 1275 * We just need the target to call irq_exit() and re-evaluate 1276 * the next tick. The nohz full kick at least implies that. 1277 * If needed we can still optimize that later with an 1278 * empty IRQ. 1279 */ 1280 if (cpu_is_offline(cpu)) 1281 return true; /* Don't try to wake offline CPUs. */ 1282 if (tick_nohz_full_cpu(cpu)) { 1283 if (cpu != smp_processor_id() || 1284 tick_nohz_tick_stopped()) 1285 tick_nohz_full_kick_cpu(cpu); 1286 return true; 1287 } 1288 1289 return false; 1290 } 1291 1292 /* 1293 * Wake up the specified CPU. If the CPU is going offline, it is the 1294 * caller's responsibility to deal with the lost wakeup, for example, 1295 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1296 */ 1297 void wake_up_nohz_cpu(int cpu) 1298 { 1299 if (!wake_up_full_nohz_cpu(cpu)) 1300 wake_up_idle_cpu(cpu); 1301 } 1302 1303 static void nohz_csd_func(void *info) 1304 { 1305 struct rq *rq = info; 1306 int cpu = cpu_of(rq); 1307 unsigned int flags; 1308 1309 /* 1310 * Release the rq::nohz_csd. 1311 */ 1312 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1313 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1314 1315 rq->idle_balance = idle_cpu(cpu); 1316 if (rq->idle_balance) { 1317 rq->nohz_idle_balance = flags; 1318 __raise_softirq_irqoff(SCHED_SOFTIRQ); 1319 } 1320 } 1321 1322 #endif /* CONFIG_NO_HZ_COMMON */ 1323 1324 #ifdef CONFIG_NO_HZ_FULL 1325 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) 1326 { 1327 if (rq->nr_running != 1) 1328 return false; 1329 1330 if (p->sched_class != &fair_sched_class) 1331 return false; 1332 1333 if (!task_on_rq_queued(p)) 1334 return false; 1335 1336 return true; 1337 } 1338 1339 bool sched_can_stop_tick(struct rq *rq) 1340 { 1341 int fifo_nr_running; 1342 1343 /* Deadline tasks, even if single, need the tick */ 1344 if (rq->dl.dl_nr_running) 1345 return false; 1346 1347 /* 1348 * If there are more than one RR tasks, we need the tick to affect the 1349 * actual RR behaviour. 1350 */ 1351 if (rq->rt.rr_nr_running) { 1352 if (rq->rt.rr_nr_running == 1) 1353 return true; 1354 else 1355 return false; 1356 } 1357 1358 /* 1359 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1360 * forced preemption between FIFO tasks. 1361 */ 1362 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1363 if (fifo_nr_running) 1364 return true; 1365 1366 /* 1367 * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks 1368 * left. For CFS, if there's more than one we need the tick for 1369 * involuntary preemption. For SCX, ask. 1370 */ 1371 if (scx_enabled() && !scx_can_stop_tick(rq)) 1372 return false; 1373 1374 if (rq->cfs.h_nr_queued > 1) 1375 return false; 1376 1377 /* 1378 * If there is one task and it has CFS runtime bandwidth constraints 1379 * and it's on the cpu now we don't want to stop the tick. 1380 * This check prevents clearing the bit if a newly enqueued task here is 1381 * dequeued by migrating while the constrained task continues to run. 1382 * E.g. going from 2->1 without going through pick_next_task(). 1383 */ 1384 if (__need_bw_check(rq, rq->curr)) { 1385 if (cfs_task_bw_constrained(rq->curr)) 1386 return false; 1387 } 1388 1389 return true; 1390 } 1391 #endif /* CONFIG_NO_HZ_FULL */ 1392 1393 #if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_FAIR_GROUP_SCHED) 1394 /* 1395 * Iterate task_group tree rooted at *from, calling @down when first entering a 1396 * node and @up when leaving it for the final time. 1397 * 1398 * Caller must hold rcu_lock or sufficient equivalent. 1399 */ 1400 int walk_tg_tree_from(struct task_group *from, 1401 tg_visitor down, tg_visitor up, void *data) 1402 { 1403 struct task_group *parent, *child; 1404 int ret; 1405 1406 parent = from; 1407 1408 down: 1409 ret = (*down)(parent, data); 1410 if (ret) 1411 goto out; 1412 list_for_each_entry_rcu(child, &parent->children, siblings) { 1413 parent = child; 1414 goto down; 1415 1416 up: 1417 continue; 1418 } 1419 ret = (*up)(parent, data); 1420 if (ret || parent == from) 1421 goto out; 1422 1423 child = parent; 1424 parent = parent->parent; 1425 if (parent) 1426 goto up; 1427 out: 1428 return ret; 1429 } 1430 1431 int tg_nop(struct task_group *tg, void *data) 1432 { 1433 return 0; 1434 } 1435 #endif 1436 1437 void set_load_weight(struct task_struct *p, bool update_load) 1438 { 1439 int prio = p->static_prio - MAX_RT_PRIO; 1440 struct load_weight lw; 1441 1442 if (task_has_idle_policy(p)) { 1443 lw.weight = scale_load(WEIGHT_IDLEPRIO); 1444 lw.inv_weight = WMULT_IDLEPRIO; 1445 } else { 1446 lw.weight = scale_load(sched_prio_to_weight[prio]); 1447 lw.inv_weight = sched_prio_to_wmult[prio]; 1448 } 1449 1450 /* 1451 * SCHED_OTHER tasks have to update their load when changing their 1452 * weight 1453 */ 1454 if (update_load && p->sched_class->reweight_task) 1455 p->sched_class->reweight_task(task_rq(p), p, &lw); 1456 else 1457 p->se.load = lw; 1458 } 1459 1460 #ifdef CONFIG_UCLAMP_TASK 1461 /* 1462 * Serializes updates of utilization clamp values 1463 * 1464 * The (slow-path) user-space triggers utilization clamp value updates which 1465 * can require updates on (fast-path) scheduler's data structures used to 1466 * support enqueue/dequeue operations. 1467 * While the per-CPU rq lock protects fast-path update operations, user-space 1468 * requests are serialized using a mutex to reduce the risk of conflicting 1469 * updates or API abuses. 1470 */ 1471 static __maybe_unused DEFINE_MUTEX(uclamp_mutex); 1472 1473 /* Max allowed minimum utilization */ 1474 static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1475 1476 /* Max allowed maximum utilization */ 1477 static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1478 1479 /* 1480 * By default RT tasks run at the maximum performance point/capacity of the 1481 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1482 * SCHED_CAPACITY_SCALE. 1483 * 1484 * This knob allows admins to change the default behavior when uclamp is being 1485 * used. In battery powered devices, particularly, running at the maximum 1486 * capacity and frequency will increase energy consumption and shorten the 1487 * battery life. 1488 * 1489 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1490 * 1491 * This knob will not override the system default sched_util_clamp_min defined 1492 * above. 1493 */ 1494 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1495 1496 /* All clamps are required to be less or equal than these values */ 1497 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1498 1499 /* 1500 * This static key is used to reduce the uclamp overhead in the fast path. It 1501 * primarily disables the call to uclamp_rq_{inc, dec}() in 1502 * enqueue/dequeue_task(). 1503 * 1504 * This allows users to continue to enable uclamp in their kernel config with 1505 * minimum uclamp overhead in the fast path. 1506 * 1507 * As soon as userspace modifies any of the uclamp knobs, the static key is 1508 * enabled, since we have an actual users that make use of uclamp 1509 * functionality. 1510 * 1511 * The knobs that would enable this static key are: 1512 * 1513 * * A task modifying its uclamp value with sched_setattr(). 1514 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1515 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1516 */ 1517 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1518 1519 static inline unsigned int 1520 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1521 unsigned int clamp_value) 1522 { 1523 /* 1524 * Avoid blocked utilization pushing up the frequency when we go 1525 * idle (which drops the max-clamp) by retaining the last known 1526 * max-clamp. 1527 */ 1528 if (clamp_id == UCLAMP_MAX) { 1529 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1530 return clamp_value; 1531 } 1532 1533 return uclamp_none(UCLAMP_MIN); 1534 } 1535 1536 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1537 unsigned int clamp_value) 1538 { 1539 /* Reset max-clamp retention only on idle exit */ 1540 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1541 return; 1542 1543 uclamp_rq_set(rq, clamp_id, clamp_value); 1544 } 1545 1546 static inline 1547 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1548 unsigned int clamp_value) 1549 { 1550 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1551 int bucket_id = UCLAMP_BUCKETS - 1; 1552 1553 /* 1554 * Since both min and max clamps are max aggregated, find the 1555 * top most bucket with tasks in. 1556 */ 1557 for ( ; bucket_id >= 0; bucket_id--) { 1558 if (!bucket[bucket_id].tasks) 1559 continue; 1560 return bucket[bucket_id].value; 1561 } 1562 1563 /* No tasks -- default clamp values */ 1564 return uclamp_idle_value(rq, clamp_id, clamp_value); 1565 } 1566 1567 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1568 { 1569 unsigned int default_util_min; 1570 struct uclamp_se *uc_se; 1571 1572 lockdep_assert_held(&p->pi_lock); 1573 1574 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1575 1576 /* Only sync if user didn't override the default */ 1577 if (uc_se->user_defined) 1578 return; 1579 1580 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1581 uclamp_se_set(uc_se, default_util_min, false); 1582 } 1583 1584 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1585 { 1586 if (!rt_task(p)) 1587 return; 1588 1589 /* Protect updates to p->uclamp_* */ 1590 guard(task_rq_lock)(p); 1591 __uclamp_update_util_min_rt_default(p); 1592 } 1593 1594 static inline struct uclamp_se 1595 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1596 { 1597 /* Copy by value as we could modify it */ 1598 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1599 #ifdef CONFIG_UCLAMP_TASK_GROUP 1600 unsigned int tg_min, tg_max, value; 1601 1602 /* 1603 * Tasks in autogroups or root task group will be 1604 * restricted by system defaults. 1605 */ 1606 if (task_group_is_autogroup(task_group(p))) 1607 return uc_req; 1608 if (task_group(p) == &root_task_group) 1609 return uc_req; 1610 1611 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1612 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1613 value = uc_req.value; 1614 value = clamp(value, tg_min, tg_max); 1615 uclamp_se_set(&uc_req, value, false); 1616 #endif 1617 1618 return uc_req; 1619 } 1620 1621 /* 1622 * The effective clamp bucket index of a task depends on, by increasing 1623 * priority: 1624 * - the task specific clamp value, when explicitly requested from userspace 1625 * - the task group effective clamp value, for tasks not either in the root 1626 * group or in an autogroup 1627 * - the system default clamp value, defined by the sysadmin 1628 */ 1629 static inline struct uclamp_se 1630 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1631 { 1632 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1633 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1634 1635 /* System default restrictions always apply */ 1636 if (unlikely(uc_req.value > uc_max.value)) 1637 return uc_max; 1638 1639 return uc_req; 1640 } 1641 1642 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1643 { 1644 struct uclamp_se uc_eff; 1645 1646 /* Task currently refcounted: use back-annotated (effective) value */ 1647 if (p->uclamp[clamp_id].active) 1648 return (unsigned long)p->uclamp[clamp_id].value; 1649 1650 uc_eff = uclamp_eff_get(p, clamp_id); 1651 1652 return (unsigned long)uc_eff.value; 1653 } 1654 1655 /* 1656 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1657 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1658 * updates the rq's clamp value if required. 1659 * 1660 * Tasks can have a task-specific value requested from user-space, track 1661 * within each bucket the maximum value for tasks refcounted in it. 1662 * This "local max aggregation" allows to track the exact "requested" value 1663 * for each bucket when all its RUNNABLE tasks require the same clamp. 1664 */ 1665 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1666 enum uclamp_id clamp_id) 1667 { 1668 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1669 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1670 struct uclamp_bucket *bucket; 1671 1672 lockdep_assert_rq_held(rq); 1673 1674 /* Update task effective clamp */ 1675 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1676 1677 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1678 bucket->tasks++; 1679 uc_se->active = true; 1680 1681 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1682 1683 /* 1684 * Local max aggregation: rq buckets always track the max 1685 * "requested" clamp value of its RUNNABLE tasks. 1686 */ 1687 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1688 bucket->value = uc_se->value; 1689 1690 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) 1691 uclamp_rq_set(rq, clamp_id, uc_se->value); 1692 } 1693 1694 /* 1695 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1696 * is released. If this is the last task reference counting the rq's max 1697 * active clamp value, then the rq's clamp value is updated. 1698 * 1699 * Both refcounted tasks and rq's cached clamp values are expected to be 1700 * always valid. If it's detected they are not, as defensive programming, 1701 * enforce the expected state and warn. 1702 */ 1703 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1704 enum uclamp_id clamp_id) 1705 { 1706 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1707 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1708 struct uclamp_bucket *bucket; 1709 unsigned int bkt_clamp; 1710 unsigned int rq_clamp; 1711 1712 lockdep_assert_rq_held(rq); 1713 1714 /* 1715 * If sched_uclamp_used was enabled after task @p was enqueued, 1716 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1717 * 1718 * In this case the uc_se->active flag should be false since no uclamp 1719 * accounting was performed at enqueue time and we can just return 1720 * here. 1721 * 1722 * Need to be careful of the following enqueue/dequeue ordering 1723 * problem too 1724 * 1725 * enqueue(taskA) 1726 * // sched_uclamp_used gets enabled 1727 * enqueue(taskB) 1728 * dequeue(taskA) 1729 * // Must not decrement bucket->tasks here 1730 * dequeue(taskB) 1731 * 1732 * where we could end up with stale data in uc_se and 1733 * bucket[uc_se->bucket_id]. 1734 * 1735 * The following check here eliminates the possibility of such race. 1736 */ 1737 if (unlikely(!uc_se->active)) 1738 return; 1739 1740 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1741 1742 WARN_ON_ONCE(!bucket->tasks); 1743 if (likely(bucket->tasks)) 1744 bucket->tasks--; 1745 1746 uc_se->active = false; 1747 1748 /* 1749 * Keep "local max aggregation" simple and accept to (possibly) 1750 * overboost some RUNNABLE tasks in the same bucket. 1751 * The rq clamp bucket value is reset to its base value whenever 1752 * there are no more RUNNABLE tasks refcounting it. 1753 */ 1754 if (likely(bucket->tasks)) 1755 return; 1756 1757 rq_clamp = uclamp_rq_get(rq, clamp_id); 1758 /* 1759 * Defensive programming: this should never happen. If it happens, 1760 * e.g. due to future modification, warn and fix up the expected value. 1761 */ 1762 WARN_ON_ONCE(bucket->value > rq_clamp); 1763 if (bucket->value >= rq_clamp) { 1764 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1765 uclamp_rq_set(rq, clamp_id, bkt_clamp); 1766 } 1767 } 1768 1769 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) 1770 { 1771 enum uclamp_id clamp_id; 1772 1773 /* 1774 * Avoid any overhead until uclamp is actually used by the userspace. 1775 * 1776 * The condition is constructed such that a NOP is generated when 1777 * sched_uclamp_used is disabled. 1778 */ 1779 if (!uclamp_is_used()) 1780 return; 1781 1782 if (unlikely(!p->sched_class->uclamp_enabled)) 1783 return; 1784 1785 /* Only inc the delayed task which being woken up. */ 1786 if (p->se.sched_delayed && !(flags & ENQUEUE_DELAYED)) 1787 return; 1788 1789 for_each_clamp_id(clamp_id) 1790 uclamp_rq_inc_id(rq, p, clamp_id); 1791 1792 /* Reset clamp idle holding when there is one RUNNABLE task */ 1793 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1794 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1795 } 1796 1797 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1798 { 1799 enum uclamp_id clamp_id; 1800 1801 /* 1802 * Avoid any overhead until uclamp is actually used by the userspace. 1803 * 1804 * The condition is constructed such that a NOP is generated when 1805 * sched_uclamp_used is disabled. 1806 */ 1807 if (!uclamp_is_used()) 1808 return; 1809 1810 if (unlikely(!p->sched_class->uclamp_enabled)) 1811 return; 1812 1813 if (p->se.sched_delayed) 1814 return; 1815 1816 for_each_clamp_id(clamp_id) 1817 uclamp_rq_dec_id(rq, p, clamp_id); 1818 } 1819 1820 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, 1821 enum uclamp_id clamp_id) 1822 { 1823 if (!p->uclamp[clamp_id].active) 1824 return; 1825 1826 uclamp_rq_dec_id(rq, p, clamp_id); 1827 uclamp_rq_inc_id(rq, p, clamp_id); 1828 1829 /* 1830 * Make sure to clear the idle flag if we've transiently reached 0 1831 * active tasks on rq. 1832 */ 1833 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1834 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1835 } 1836 1837 static inline void 1838 uclamp_update_active(struct task_struct *p) 1839 { 1840 enum uclamp_id clamp_id; 1841 struct rq_flags rf; 1842 struct rq *rq; 1843 1844 /* 1845 * Lock the task and the rq where the task is (or was) queued. 1846 * 1847 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1848 * price to pay to safely serialize util_{min,max} updates with 1849 * enqueues, dequeues and migration operations. 1850 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1851 */ 1852 rq = task_rq_lock(p, &rf); 1853 1854 /* 1855 * Setting the clamp bucket is serialized by task_rq_lock(). 1856 * If the task is not yet RUNNABLE and its task_struct is not 1857 * affecting a valid clamp bucket, the next time it's enqueued, 1858 * it will already see the updated clamp bucket value. 1859 */ 1860 for_each_clamp_id(clamp_id) 1861 uclamp_rq_reinc_id(rq, p, clamp_id); 1862 1863 task_rq_unlock(rq, p, &rf); 1864 } 1865 1866 #ifdef CONFIG_UCLAMP_TASK_GROUP 1867 static inline void 1868 uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1869 { 1870 struct css_task_iter it; 1871 struct task_struct *p; 1872 1873 css_task_iter_start(css, 0, &it); 1874 while ((p = css_task_iter_next(&it))) 1875 uclamp_update_active(p); 1876 css_task_iter_end(&it); 1877 } 1878 1879 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1880 #endif 1881 1882 #ifdef CONFIG_SYSCTL 1883 #ifdef CONFIG_UCLAMP_TASK_GROUP 1884 static void uclamp_update_root_tg(void) 1885 { 1886 struct task_group *tg = &root_task_group; 1887 1888 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1889 sysctl_sched_uclamp_util_min, false); 1890 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1891 sysctl_sched_uclamp_util_max, false); 1892 1893 guard(rcu)(); 1894 cpu_util_update_eff(&root_task_group.css); 1895 } 1896 #else 1897 static void uclamp_update_root_tg(void) { } 1898 #endif 1899 1900 static void uclamp_sync_util_min_rt_default(void) 1901 { 1902 struct task_struct *g, *p; 1903 1904 /* 1905 * copy_process() sysctl_uclamp 1906 * uclamp_min_rt = X; 1907 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1908 * // link thread smp_mb__after_spinlock() 1909 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1910 * sched_post_fork() for_each_process_thread() 1911 * __uclamp_sync_rt() __uclamp_sync_rt() 1912 * 1913 * Ensures that either sched_post_fork() will observe the new 1914 * uclamp_min_rt or for_each_process_thread() will observe the new 1915 * task. 1916 */ 1917 read_lock(&tasklist_lock); 1918 smp_mb__after_spinlock(); 1919 read_unlock(&tasklist_lock); 1920 1921 guard(rcu)(); 1922 for_each_process_thread(g, p) 1923 uclamp_update_util_min_rt_default(p); 1924 } 1925 1926 static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write, 1927 void *buffer, size_t *lenp, loff_t *ppos) 1928 { 1929 bool update_root_tg = false; 1930 int old_min, old_max, old_min_rt; 1931 int result; 1932 1933 guard(mutex)(&uclamp_mutex); 1934 1935 old_min = sysctl_sched_uclamp_util_min; 1936 old_max = sysctl_sched_uclamp_util_max; 1937 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1938 1939 result = proc_dointvec(table, write, buffer, lenp, ppos); 1940 if (result) 1941 goto undo; 1942 if (!write) 1943 return 0; 1944 1945 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1946 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1947 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1948 1949 result = -EINVAL; 1950 goto undo; 1951 } 1952 1953 if (old_min != sysctl_sched_uclamp_util_min) { 1954 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1955 sysctl_sched_uclamp_util_min, false); 1956 update_root_tg = true; 1957 } 1958 if (old_max != sysctl_sched_uclamp_util_max) { 1959 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1960 sysctl_sched_uclamp_util_max, false); 1961 update_root_tg = true; 1962 } 1963 1964 if (update_root_tg) { 1965 sched_uclamp_enable(); 1966 uclamp_update_root_tg(); 1967 } 1968 1969 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1970 sched_uclamp_enable(); 1971 uclamp_sync_util_min_rt_default(); 1972 } 1973 1974 /* 1975 * We update all RUNNABLE tasks only when task groups are in use. 1976 * Otherwise, keep it simple and do just a lazy update at each next 1977 * task enqueue time. 1978 */ 1979 return 0; 1980 1981 undo: 1982 sysctl_sched_uclamp_util_min = old_min; 1983 sysctl_sched_uclamp_util_max = old_max; 1984 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1985 return result; 1986 } 1987 #endif /* CONFIG_SYSCTL */ 1988 1989 static void uclamp_fork(struct task_struct *p) 1990 { 1991 enum uclamp_id clamp_id; 1992 1993 /* 1994 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1995 * as the task is still at its early fork stages. 1996 */ 1997 for_each_clamp_id(clamp_id) 1998 p->uclamp[clamp_id].active = false; 1999 2000 if (likely(!p->sched_reset_on_fork)) 2001 return; 2002 2003 for_each_clamp_id(clamp_id) { 2004 uclamp_se_set(&p->uclamp_req[clamp_id], 2005 uclamp_none(clamp_id), false); 2006 } 2007 } 2008 2009 static void uclamp_post_fork(struct task_struct *p) 2010 { 2011 uclamp_update_util_min_rt_default(p); 2012 } 2013 2014 static void __init init_uclamp_rq(struct rq *rq) 2015 { 2016 enum uclamp_id clamp_id; 2017 struct uclamp_rq *uc_rq = rq->uclamp; 2018 2019 for_each_clamp_id(clamp_id) { 2020 uc_rq[clamp_id] = (struct uclamp_rq) { 2021 .value = uclamp_none(clamp_id) 2022 }; 2023 } 2024 2025 rq->uclamp_flags = UCLAMP_FLAG_IDLE; 2026 } 2027 2028 static void __init init_uclamp(void) 2029 { 2030 struct uclamp_se uc_max = {}; 2031 enum uclamp_id clamp_id; 2032 int cpu; 2033 2034 for_each_possible_cpu(cpu) 2035 init_uclamp_rq(cpu_rq(cpu)); 2036 2037 for_each_clamp_id(clamp_id) { 2038 uclamp_se_set(&init_task.uclamp_req[clamp_id], 2039 uclamp_none(clamp_id), false); 2040 } 2041 2042 /* System defaults allow max clamp values for both indexes */ 2043 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 2044 for_each_clamp_id(clamp_id) { 2045 uclamp_default[clamp_id] = uc_max; 2046 #ifdef CONFIG_UCLAMP_TASK_GROUP 2047 root_task_group.uclamp_req[clamp_id] = uc_max; 2048 root_task_group.uclamp[clamp_id] = uc_max; 2049 #endif 2050 } 2051 } 2052 2053 #else /* !CONFIG_UCLAMP_TASK: */ 2054 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) { } 2055 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 2056 static inline void uclamp_fork(struct task_struct *p) { } 2057 static inline void uclamp_post_fork(struct task_struct *p) { } 2058 static inline void init_uclamp(void) { } 2059 #endif /* !CONFIG_UCLAMP_TASK */ 2060 2061 bool sched_task_on_rq(struct task_struct *p) 2062 { 2063 return task_on_rq_queued(p); 2064 } 2065 2066 unsigned long get_wchan(struct task_struct *p) 2067 { 2068 unsigned long ip = 0; 2069 unsigned int state; 2070 2071 if (!p || p == current) 2072 return 0; 2073 2074 /* Only get wchan if task is blocked and we can keep it that way. */ 2075 raw_spin_lock_irq(&p->pi_lock); 2076 state = READ_ONCE(p->__state); 2077 smp_rmb(); /* see try_to_wake_up() */ 2078 if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) 2079 ip = __get_wchan(p); 2080 raw_spin_unlock_irq(&p->pi_lock); 2081 2082 return ip; 2083 } 2084 2085 void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 2086 { 2087 if (!(flags & ENQUEUE_NOCLOCK)) 2088 update_rq_clock(rq); 2089 2090 /* 2091 * Can be before ->enqueue_task() because uclamp considers the 2092 * ENQUEUE_DELAYED task before its ->sched_delayed gets cleared 2093 * in ->enqueue_task(). 2094 */ 2095 uclamp_rq_inc(rq, p, flags); 2096 2097 p->sched_class->enqueue_task(rq, p, flags); 2098 2099 psi_enqueue(p, flags); 2100 2101 if (!(flags & ENQUEUE_RESTORE)) 2102 sched_info_enqueue(rq, p); 2103 2104 if (sched_core_enabled(rq)) 2105 sched_core_enqueue(rq, p); 2106 } 2107 2108 /* 2109 * Must only return false when DEQUEUE_SLEEP. 2110 */ 2111 inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags) 2112 { 2113 if (sched_core_enabled(rq)) 2114 sched_core_dequeue(rq, p, flags); 2115 2116 if (!(flags & DEQUEUE_NOCLOCK)) 2117 update_rq_clock(rq); 2118 2119 if (!(flags & DEQUEUE_SAVE)) 2120 sched_info_dequeue(rq, p); 2121 2122 psi_dequeue(p, flags); 2123 2124 /* 2125 * Must be before ->dequeue_task() because ->dequeue_task() can 'fail' 2126 * and mark the task ->sched_delayed. 2127 */ 2128 uclamp_rq_dec(rq, p); 2129 return p->sched_class->dequeue_task(rq, p, flags); 2130 } 2131 2132 void activate_task(struct rq *rq, struct task_struct *p, int flags) 2133 { 2134 if (task_on_rq_migrating(p)) 2135 flags |= ENQUEUE_MIGRATED; 2136 if (flags & ENQUEUE_MIGRATED) 2137 sched_mm_cid_migrate_to(rq, p); 2138 2139 enqueue_task(rq, p, flags); 2140 2141 WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED); 2142 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2143 } 2144 2145 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 2146 { 2147 WARN_ON_ONCE(flags & DEQUEUE_SLEEP); 2148 2149 WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); 2150 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2151 2152 /* 2153 * Code explicitly relies on TASK_ON_RQ_MIGRATING begin set *before* 2154 * dequeue_task() and cleared *after* enqueue_task(). 2155 */ 2156 2157 dequeue_task(rq, p, flags); 2158 } 2159 2160 static void block_task(struct rq *rq, struct task_struct *p, int flags) 2161 { 2162 if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) 2163 __block_task(rq, p); 2164 } 2165 2166 /** 2167 * task_curr - is this task currently executing on a CPU? 2168 * @p: the task in question. 2169 * 2170 * Return: 1 if the task is currently executing. 0 otherwise. 2171 */ 2172 inline int task_curr(const struct task_struct *p) 2173 { 2174 return cpu_curr(task_cpu(p)) == p; 2175 } 2176 2177 /* 2178 * ->switching_to() is called with the pi_lock and rq_lock held and must not 2179 * mess with locking. 2180 */ 2181 void check_class_changing(struct rq *rq, struct task_struct *p, 2182 const struct sched_class *prev_class) 2183 { 2184 if (prev_class != p->sched_class && p->sched_class->switching_to) 2185 p->sched_class->switching_to(rq, p); 2186 } 2187 2188 /* 2189 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 2190 * use the balance_callback list if you want balancing. 2191 * 2192 * this means any call to check_class_changed() must be followed by a call to 2193 * balance_callback(). 2194 */ 2195 void check_class_changed(struct rq *rq, struct task_struct *p, 2196 const struct sched_class *prev_class, 2197 int oldprio) 2198 { 2199 if (prev_class != p->sched_class) { 2200 if (prev_class->switched_from) 2201 prev_class->switched_from(rq, p); 2202 2203 p->sched_class->switched_to(rq, p); 2204 } else if (oldprio != p->prio || dl_task(p)) 2205 p->sched_class->prio_changed(rq, p, oldprio); 2206 } 2207 2208 void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags) 2209 { 2210 struct task_struct *donor = rq->donor; 2211 2212 if (p->sched_class == donor->sched_class) 2213 donor->sched_class->wakeup_preempt(rq, p, flags); 2214 else if (sched_class_above(p->sched_class, donor->sched_class)) 2215 resched_curr(rq); 2216 2217 /* 2218 * A queue event has occurred, and we're going to schedule. In 2219 * this case, we can save a useless back to back clock update. 2220 */ 2221 if (task_on_rq_queued(donor) && test_tsk_need_resched(rq->curr)) 2222 rq_clock_skip_update(rq); 2223 } 2224 2225 static __always_inline 2226 int __task_state_match(struct task_struct *p, unsigned int state) 2227 { 2228 if (READ_ONCE(p->__state) & state) 2229 return 1; 2230 2231 if (READ_ONCE(p->saved_state) & state) 2232 return -1; 2233 2234 return 0; 2235 } 2236 2237 static __always_inline 2238 int task_state_match(struct task_struct *p, unsigned int state) 2239 { 2240 /* 2241 * Serialize against current_save_and_set_rtlock_wait_state(), 2242 * current_restore_rtlock_saved_state(), and __refrigerator(). 2243 */ 2244 guard(raw_spinlock_irq)(&p->pi_lock); 2245 return __task_state_match(p, state); 2246 } 2247 2248 /* 2249 * wait_task_inactive - wait for a thread to unschedule. 2250 * 2251 * Wait for the thread to block in any of the states set in @match_state. 2252 * If it changes, i.e. @p might have woken up, then return zero. When we 2253 * succeed in waiting for @p to be off its CPU, we return a positive number 2254 * (its total switch count). If a second call a short while later returns the 2255 * same number, the caller can be sure that @p has remained unscheduled the 2256 * whole time. 2257 * 2258 * The caller must ensure that the task *will* unschedule sometime soon, 2259 * else this function might spin for a *long* time. This function can't 2260 * be called with interrupts off, or it may introduce deadlock with 2261 * smp_call_function() if an IPI is sent by the same process we are 2262 * waiting to become inactive. 2263 */ 2264 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2265 { 2266 int running, queued, match; 2267 struct rq_flags rf; 2268 unsigned long ncsw; 2269 struct rq *rq; 2270 2271 for (;;) { 2272 /* 2273 * We do the initial early heuristics without holding 2274 * any task-queue locks at all. We'll only try to get 2275 * the runqueue lock when things look like they will 2276 * work out! 2277 */ 2278 rq = task_rq(p); 2279 2280 /* 2281 * If the task is actively running on another CPU 2282 * still, just relax and busy-wait without holding 2283 * any locks. 2284 * 2285 * NOTE! Since we don't hold any locks, it's not 2286 * even sure that "rq" stays as the right runqueue! 2287 * But we don't care, since "task_on_cpu()" will 2288 * return false if the runqueue has changed and p 2289 * is actually now running somewhere else! 2290 */ 2291 while (task_on_cpu(rq, p)) { 2292 if (!task_state_match(p, match_state)) 2293 return 0; 2294 cpu_relax(); 2295 } 2296 2297 /* 2298 * Ok, time to look more closely! We need the rq 2299 * lock now, to be *sure*. If we're wrong, we'll 2300 * just go back and repeat. 2301 */ 2302 rq = task_rq_lock(p, &rf); 2303 /* 2304 * If task is sched_delayed, force dequeue it, to avoid always 2305 * hitting the tick timeout in the queued case 2306 */ 2307 if (p->se.sched_delayed) 2308 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 2309 trace_sched_wait_task(p); 2310 running = task_on_cpu(rq, p); 2311 queued = task_on_rq_queued(p); 2312 ncsw = 0; 2313 if ((match = __task_state_match(p, match_state))) { 2314 /* 2315 * When matching on p->saved_state, consider this task 2316 * still queued so it will wait. 2317 */ 2318 if (match < 0) 2319 queued = 1; 2320 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2321 } 2322 task_rq_unlock(rq, p, &rf); 2323 2324 /* 2325 * If it changed from the expected state, bail out now. 2326 */ 2327 if (unlikely(!ncsw)) 2328 break; 2329 2330 /* 2331 * Was it really running after all now that we 2332 * checked with the proper locks actually held? 2333 * 2334 * Oops. Go back and try again.. 2335 */ 2336 if (unlikely(running)) { 2337 cpu_relax(); 2338 continue; 2339 } 2340 2341 /* 2342 * It's not enough that it's not actively running, 2343 * it must be off the runqueue _entirely_, and not 2344 * preempted! 2345 * 2346 * So if it was still runnable (but just not actively 2347 * running right now), it's preempted, and we should 2348 * yield - it could be a while. 2349 */ 2350 if (unlikely(queued)) { 2351 ktime_t to = NSEC_PER_SEC / HZ; 2352 2353 set_current_state(TASK_UNINTERRUPTIBLE); 2354 schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); 2355 continue; 2356 } 2357 2358 /* 2359 * Ahh, all good. It wasn't running, and it wasn't 2360 * runnable, which means that it will never become 2361 * running in the future either. We're all done! 2362 */ 2363 break; 2364 } 2365 2366 return ncsw; 2367 } 2368 2369 static void 2370 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); 2371 2372 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2373 { 2374 struct affinity_context ac = { 2375 .new_mask = cpumask_of(rq->cpu), 2376 .flags = SCA_MIGRATE_DISABLE, 2377 }; 2378 2379 if (likely(!p->migration_disabled)) 2380 return; 2381 2382 if (p->cpus_ptr != &p->cpus_mask) 2383 return; 2384 2385 /* 2386 * Violates locking rules! See comment in __do_set_cpus_allowed(). 2387 */ 2388 __do_set_cpus_allowed(p, &ac); 2389 } 2390 2391 void ___migrate_enable(void) 2392 { 2393 struct task_struct *p = current; 2394 struct affinity_context ac = { 2395 .new_mask = &p->cpus_mask, 2396 .flags = SCA_MIGRATE_ENABLE, 2397 }; 2398 2399 __set_cpus_allowed_ptr(p, &ac); 2400 } 2401 EXPORT_SYMBOL_GPL(___migrate_enable); 2402 2403 void migrate_disable(void) 2404 { 2405 __migrate_disable(); 2406 } 2407 EXPORT_SYMBOL_GPL(migrate_disable); 2408 2409 void migrate_enable(void) 2410 { 2411 __migrate_enable(); 2412 } 2413 EXPORT_SYMBOL_GPL(migrate_enable); 2414 2415 static inline bool rq_has_pinned_tasks(struct rq *rq) 2416 { 2417 return rq->nr_pinned; 2418 } 2419 2420 /* 2421 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2422 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2423 */ 2424 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2425 { 2426 /* When not in the task's cpumask, no point in looking further. */ 2427 if (!task_allowed_on_cpu(p, cpu)) 2428 return false; 2429 2430 /* migrate_disabled() must be allowed to finish. */ 2431 if (is_migration_disabled(p)) 2432 return cpu_online(cpu); 2433 2434 /* Non kernel threads are not allowed during either online or offline. */ 2435 if (!(p->flags & PF_KTHREAD)) 2436 return cpu_active(cpu); 2437 2438 /* KTHREAD_IS_PER_CPU is always allowed. */ 2439 if (kthread_is_per_cpu(p)) 2440 return cpu_online(cpu); 2441 2442 /* Regular kernel threads don't get to stay during offline. */ 2443 if (cpu_dying(cpu)) 2444 return false; 2445 2446 /* But are allowed during online. */ 2447 return cpu_online(cpu); 2448 } 2449 2450 /* 2451 * This is how migration works: 2452 * 2453 * 1) we invoke migration_cpu_stop() on the target CPU using 2454 * stop_one_cpu(). 2455 * 2) stopper starts to run (implicitly forcing the migrated thread 2456 * off the CPU) 2457 * 3) it checks whether the migrated task is still in the wrong runqueue. 2458 * 4) if it's in the wrong runqueue then the migration thread removes 2459 * it and puts it into the right queue. 2460 * 5) stopper completes and stop_one_cpu() returns and the migration 2461 * is done. 2462 */ 2463 2464 /* 2465 * move_queued_task - move a queued task to new rq. 2466 * 2467 * Returns (locked) new rq. Old rq's lock is released. 2468 */ 2469 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2470 struct task_struct *p, int new_cpu) 2471 { 2472 lockdep_assert_rq_held(rq); 2473 2474 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2475 set_task_cpu(p, new_cpu); 2476 rq_unlock(rq, rf); 2477 2478 rq = cpu_rq(new_cpu); 2479 2480 rq_lock(rq, rf); 2481 WARN_ON_ONCE(task_cpu(p) != new_cpu); 2482 activate_task(rq, p, 0); 2483 wakeup_preempt(rq, p, 0); 2484 2485 return rq; 2486 } 2487 2488 struct migration_arg { 2489 struct task_struct *task; 2490 int dest_cpu; 2491 struct set_affinity_pending *pending; 2492 }; 2493 2494 /* 2495 * @refs: number of wait_for_completion() 2496 * @stop_pending: is @stop_work in use 2497 */ 2498 struct set_affinity_pending { 2499 refcount_t refs; 2500 unsigned int stop_pending; 2501 struct completion done; 2502 struct cpu_stop_work stop_work; 2503 struct migration_arg arg; 2504 }; 2505 2506 /* 2507 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2508 * this because either it can't run here any more (set_cpus_allowed() 2509 * away from this CPU, or CPU going down), or because we're 2510 * attempting to rebalance this task on exec (sched_exec). 2511 * 2512 * So we race with normal scheduler movements, but that's OK, as long 2513 * as the task is no longer on this CPU. 2514 */ 2515 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2516 struct task_struct *p, int dest_cpu) 2517 { 2518 /* Affinity changed (again). */ 2519 if (!is_cpu_allowed(p, dest_cpu)) 2520 return rq; 2521 2522 rq = move_queued_task(rq, rf, p, dest_cpu); 2523 2524 return rq; 2525 } 2526 2527 /* 2528 * migration_cpu_stop - this will be executed by a high-prio stopper thread 2529 * and performs thread migration by bumping thread off CPU then 2530 * 'pushing' onto another runqueue. 2531 */ 2532 static int migration_cpu_stop(void *data) 2533 { 2534 struct migration_arg *arg = data; 2535 struct set_affinity_pending *pending = arg->pending; 2536 struct task_struct *p = arg->task; 2537 struct rq *rq = this_rq(); 2538 bool complete = false; 2539 struct rq_flags rf; 2540 2541 /* 2542 * The original target CPU might have gone down and we might 2543 * be on another CPU but it doesn't matter. 2544 */ 2545 local_irq_save(rf.flags); 2546 /* 2547 * We need to explicitly wake pending tasks before running 2548 * __migrate_task() such that we will not miss enforcing cpus_ptr 2549 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2550 */ 2551 flush_smp_call_function_queue(); 2552 2553 raw_spin_lock(&p->pi_lock); 2554 rq_lock(rq, &rf); 2555 2556 /* 2557 * If we were passed a pending, then ->stop_pending was set, thus 2558 * p->migration_pending must have remained stable. 2559 */ 2560 WARN_ON_ONCE(pending && pending != p->migration_pending); 2561 2562 /* 2563 * If task_rq(p) != rq, it cannot be migrated here, because we're 2564 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2565 * we're holding p->pi_lock. 2566 */ 2567 if (task_rq(p) == rq) { 2568 if (is_migration_disabled(p)) 2569 goto out; 2570 2571 if (pending) { 2572 p->migration_pending = NULL; 2573 complete = true; 2574 2575 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2576 goto out; 2577 } 2578 2579 if (task_on_rq_queued(p)) { 2580 update_rq_clock(rq); 2581 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2582 } else { 2583 p->wake_cpu = arg->dest_cpu; 2584 } 2585 2586 /* 2587 * XXX __migrate_task() can fail, at which point we might end 2588 * up running on a dodgy CPU, AFAICT this can only happen 2589 * during CPU hotplug, at which point we'll get pushed out 2590 * anyway, so it's probably not a big deal. 2591 */ 2592 2593 } else if (pending) { 2594 /* 2595 * This happens when we get migrated between migrate_enable()'s 2596 * preempt_enable() and scheduling the stopper task. At that 2597 * point we're a regular task again and not current anymore. 2598 * 2599 * A !PREEMPT kernel has a giant hole here, which makes it far 2600 * more likely. 2601 */ 2602 2603 /* 2604 * The task moved before the stopper got to run. We're holding 2605 * ->pi_lock, so the allowed mask is stable - if it got 2606 * somewhere allowed, we're done. 2607 */ 2608 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2609 p->migration_pending = NULL; 2610 complete = true; 2611 goto out; 2612 } 2613 2614 /* 2615 * When migrate_enable() hits a rq mis-match we can't reliably 2616 * determine is_migration_disabled() and so have to chase after 2617 * it. 2618 */ 2619 WARN_ON_ONCE(!pending->stop_pending); 2620 preempt_disable(); 2621 task_rq_unlock(rq, p, &rf); 2622 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2623 &pending->arg, &pending->stop_work); 2624 preempt_enable(); 2625 return 0; 2626 } 2627 out: 2628 if (pending) 2629 pending->stop_pending = false; 2630 task_rq_unlock(rq, p, &rf); 2631 2632 if (complete) 2633 complete_all(&pending->done); 2634 2635 return 0; 2636 } 2637 2638 int push_cpu_stop(void *arg) 2639 { 2640 struct rq *lowest_rq = NULL, *rq = this_rq(); 2641 struct task_struct *p = arg; 2642 2643 raw_spin_lock_irq(&p->pi_lock); 2644 raw_spin_rq_lock(rq); 2645 2646 if (task_rq(p) != rq) 2647 goto out_unlock; 2648 2649 if (is_migration_disabled(p)) { 2650 p->migration_flags |= MDF_PUSH; 2651 goto out_unlock; 2652 } 2653 2654 p->migration_flags &= ~MDF_PUSH; 2655 2656 if (p->sched_class->find_lock_rq) 2657 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2658 2659 if (!lowest_rq) 2660 goto out_unlock; 2661 2662 // XXX validate p is still the highest prio task 2663 if (task_rq(p) == rq) { 2664 move_queued_task_locked(rq, lowest_rq, p); 2665 resched_curr(lowest_rq); 2666 } 2667 2668 double_unlock_balance(rq, lowest_rq); 2669 2670 out_unlock: 2671 rq->push_busy = false; 2672 raw_spin_rq_unlock(rq); 2673 raw_spin_unlock_irq(&p->pi_lock); 2674 2675 put_task_struct(p); 2676 return 0; 2677 } 2678 2679 /* 2680 * sched_class::set_cpus_allowed must do the below, but is not required to 2681 * actually call this function. 2682 */ 2683 void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) 2684 { 2685 if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2686 p->cpus_ptr = ctx->new_mask; 2687 return; 2688 } 2689 2690 cpumask_copy(&p->cpus_mask, ctx->new_mask); 2691 p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); 2692 2693 /* 2694 * Swap in a new user_cpus_ptr if SCA_USER flag set 2695 */ 2696 if (ctx->flags & SCA_USER) 2697 swap(p->user_cpus_ptr, ctx->user_mask); 2698 } 2699 2700 static void 2701 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) 2702 { 2703 struct rq *rq = task_rq(p); 2704 bool queued, running; 2705 2706 /* 2707 * This here violates the locking rules for affinity, since we're only 2708 * supposed to change these variables while holding both rq->lock and 2709 * p->pi_lock. 2710 * 2711 * HOWEVER, it magically works, because ttwu() is the only code that 2712 * accesses these variables under p->pi_lock and only does so after 2713 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() 2714 * before finish_task(). 2715 * 2716 * XXX do further audits, this smells like something putrid. 2717 */ 2718 if (ctx->flags & SCA_MIGRATE_DISABLE) 2719 WARN_ON_ONCE(!p->on_cpu); 2720 else 2721 lockdep_assert_held(&p->pi_lock); 2722 2723 queued = task_on_rq_queued(p); 2724 running = task_current_donor(rq, p); 2725 2726 if (queued) { 2727 /* 2728 * Because __kthread_bind() calls this on blocked tasks without 2729 * holding rq->lock. 2730 */ 2731 lockdep_assert_rq_held(rq); 2732 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 2733 } 2734 if (running) 2735 put_prev_task(rq, p); 2736 2737 p->sched_class->set_cpus_allowed(p, ctx); 2738 mm_set_cpus_allowed(p->mm, ctx->new_mask); 2739 2740 if (queued) 2741 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 2742 if (running) 2743 set_next_task(rq, p); 2744 } 2745 2746 /* 2747 * Used for kthread_bind() and select_fallback_rq(), in both cases the user 2748 * affinity (if any) should be destroyed too. 2749 */ 2750 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 2751 { 2752 struct affinity_context ac = { 2753 .new_mask = new_mask, 2754 .user_mask = NULL, 2755 .flags = SCA_USER, /* clear the user requested mask */ 2756 }; 2757 union cpumask_rcuhead { 2758 cpumask_t cpumask; 2759 struct rcu_head rcu; 2760 }; 2761 2762 __do_set_cpus_allowed(p, &ac); 2763 2764 /* 2765 * Because this is called with p->pi_lock held, it is not possible 2766 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using 2767 * kfree_rcu(). 2768 */ 2769 kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); 2770 } 2771 2772 int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, 2773 int node) 2774 { 2775 cpumask_t *user_mask; 2776 unsigned long flags; 2777 2778 /* 2779 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's 2780 * may differ by now due to racing. 2781 */ 2782 dst->user_cpus_ptr = NULL; 2783 2784 /* 2785 * This check is racy and losing the race is a valid situation. 2786 * It is not worth the extra overhead of taking the pi_lock on 2787 * every fork/clone. 2788 */ 2789 if (data_race(!src->user_cpus_ptr)) 2790 return 0; 2791 2792 user_mask = alloc_user_cpus_ptr(node); 2793 if (!user_mask) 2794 return -ENOMEM; 2795 2796 /* 2797 * Use pi_lock to protect content of user_cpus_ptr 2798 * 2799 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent 2800 * do_set_cpus_allowed(). 2801 */ 2802 raw_spin_lock_irqsave(&src->pi_lock, flags); 2803 if (src->user_cpus_ptr) { 2804 swap(dst->user_cpus_ptr, user_mask); 2805 cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); 2806 } 2807 raw_spin_unlock_irqrestore(&src->pi_lock, flags); 2808 2809 if (unlikely(user_mask)) 2810 kfree(user_mask); 2811 2812 return 0; 2813 } 2814 2815 static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) 2816 { 2817 struct cpumask *user_mask = NULL; 2818 2819 swap(p->user_cpus_ptr, user_mask); 2820 2821 return user_mask; 2822 } 2823 2824 void release_user_cpus_ptr(struct task_struct *p) 2825 { 2826 kfree(clear_user_cpus_ptr(p)); 2827 } 2828 2829 /* 2830 * This function is wildly self concurrent; here be dragons. 2831 * 2832 * 2833 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2834 * designated task is enqueued on an allowed CPU. If that task is currently 2835 * running, we have to kick it out using the CPU stopper. 2836 * 2837 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2838 * Consider: 2839 * 2840 * Initial conditions: P0->cpus_mask = [0, 1] 2841 * 2842 * P0@CPU0 P1 2843 * 2844 * migrate_disable(); 2845 * <preempted> 2846 * set_cpus_allowed_ptr(P0, [1]); 2847 * 2848 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2849 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2850 * This means we need the following scheme: 2851 * 2852 * P0@CPU0 P1 2853 * 2854 * migrate_disable(); 2855 * <preempted> 2856 * set_cpus_allowed_ptr(P0, [1]); 2857 * <blocks> 2858 * <resumes> 2859 * migrate_enable(); 2860 * __set_cpus_allowed_ptr(); 2861 * <wakes local stopper> 2862 * `--> <woken on migration completion> 2863 * 2864 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2865 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2866 * task p are serialized by p->pi_lock, which we can leverage: the one that 2867 * should come into effect at the end of the Migrate-Disable region is the last 2868 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2869 * but we still need to properly signal those waiting tasks at the appropriate 2870 * moment. 2871 * 2872 * This is implemented using struct set_affinity_pending. The first 2873 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2874 * setup an instance of that struct and install it on the targeted task_struct. 2875 * Any and all further callers will reuse that instance. Those then wait for 2876 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2877 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2878 * 2879 * 2880 * (1) In the cases covered above. There is one more where the completion is 2881 * signaled within affine_move_task() itself: when a subsequent affinity request 2882 * occurs after the stopper bailed out due to the targeted task still being 2883 * Migrate-Disable. Consider: 2884 * 2885 * Initial conditions: P0->cpus_mask = [0, 1] 2886 * 2887 * CPU0 P1 P2 2888 * <P0> 2889 * migrate_disable(); 2890 * <preempted> 2891 * set_cpus_allowed_ptr(P0, [1]); 2892 * <blocks> 2893 * <migration/0> 2894 * migration_cpu_stop() 2895 * is_migration_disabled() 2896 * <bails> 2897 * set_cpus_allowed_ptr(P0, [0, 1]); 2898 * <signal completion> 2899 * <awakes> 2900 * 2901 * Note that the above is safe vs a concurrent migrate_enable(), as any 2902 * pending affinity completion is preceded by an uninstallation of 2903 * p->migration_pending done with p->pi_lock held. 2904 */ 2905 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2906 int dest_cpu, unsigned int flags) 2907 __releases(rq->lock) 2908 __releases(p->pi_lock) 2909 { 2910 struct set_affinity_pending my_pending = { }, *pending = NULL; 2911 bool stop_pending, complete = false; 2912 2913 /* 2914 * Can the task run on the task's current CPU? If so, we're done 2915 * 2916 * We are also done if the task is the current donor, boosting a lock- 2917 * holding proxy, (and potentially has been migrated outside its 2918 * current or previous affinity mask) 2919 */ 2920 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask) || 2921 (task_current_donor(rq, p) && !task_current(rq, p))) { 2922 struct task_struct *push_task = NULL; 2923 2924 if ((flags & SCA_MIGRATE_ENABLE) && 2925 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2926 rq->push_busy = true; 2927 push_task = get_task_struct(p); 2928 } 2929 2930 /* 2931 * If there are pending waiters, but no pending stop_work, 2932 * then complete now. 2933 */ 2934 pending = p->migration_pending; 2935 if (pending && !pending->stop_pending) { 2936 p->migration_pending = NULL; 2937 complete = true; 2938 } 2939 2940 preempt_disable(); 2941 task_rq_unlock(rq, p, rf); 2942 if (push_task) { 2943 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2944 p, &rq->push_work); 2945 } 2946 preempt_enable(); 2947 2948 if (complete) 2949 complete_all(&pending->done); 2950 2951 return 0; 2952 } 2953 2954 if (!(flags & SCA_MIGRATE_ENABLE)) { 2955 /* serialized by p->pi_lock */ 2956 if (!p->migration_pending) { 2957 /* Install the request */ 2958 refcount_set(&my_pending.refs, 1); 2959 init_completion(&my_pending.done); 2960 my_pending.arg = (struct migration_arg) { 2961 .task = p, 2962 .dest_cpu = dest_cpu, 2963 .pending = &my_pending, 2964 }; 2965 2966 p->migration_pending = &my_pending; 2967 } else { 2968 pending = p->migration_pending; 2969 refcount_inc(&pending->refs); 2970 /* 2971 * Affinity has changed, but we've already installed a 2972 * pending. migration_cpu_stop() *must* see this, else 2973 * we risk a completion of the pending despite having a 2974 * task on a disallowed CPU. 2975 * 2976 * Serialized by p->pi_lock, so this is safe. 2977 */ 2978 pending->arg.dest_cpu = dest_cpu; 2979 } 2980 } 2981 pending = p->migration_pending; 2982 /* 2983 * - !MIGRATE_ENABLE: 2984 * we'll have installed a pending if there wasn't one already. 2985 * 2986 * - MIGRATE_ENABLE: 2987 * we're here because the current CPU isn't matching anymore, 2988 * the only way that can happen is because of a concurrent 2989 * set_cpus_allowed_ptr() call, which should then still be 2990 * pending completion. 2991 * 2992 * Either way, we really should have a @pending here. 2993 */ 2994 if (WARN_ON_ONCE(!pending)) { 2995 task_rq_unlock(rq, p, rf); 2996 return -EINVAL; 2997 } 2998 2999 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 3000 /* 3001 * MIGRATE_ENABLE gets here because 'p == current', but for 3002 * anything else we cannot do is_migration_disabled(), punt 3003 * and have the stopper function handle it all race-free. 3004 */ 3005 stop_pending = pending->stop_pending; 3006 if (!stop_pending) 3007 pending->stop_pending = true; 3008 3009 if (flags & SCA_MIGRATE_ENABLE) 3010 p->migration_flags &= ~MDF_PUSH; 3011 3012 preempt_disable(); 3013 task_rq_unlock(rq, p, rf); 3014 if (!stop_pending) { 3015 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 3016 &pending->arg, &pending->stop_work); 3017 } 3018 preempt_enable(); 3019 3020 if (flags & SCA_MIGRATE_ENABLE) 3021 return 0; 3022 } else { 3023 3024 if (!is_migration_disabled(p)) { 3025 if (task_on_rq_queued(p)) 3026 rq = move_queued_task(rq, rf, p, dest_cpu); 3027 3028 if (!pending->stop_pending) { 3029 p->migration_pending = NULL; 3030 complete = true; 3031 } 3032 } 3033 task_rq_unlock(rq, p, rf); 3034 3035 if (complete) 3036 complete_all(&pending->done); 3037 } 3038 3039 wait_for_completion(&pending->done); 3040 3041 if (refcount_dec_and_test(&pending->refs)) 3042 wake_up_var(&pending->refs); /* No UaF, just an address */ 3043 3044 /* 3045 * Block the original owner of &pending until all subsequent callers 3046 * have seen the completion and decremented the refcount 3047 */ 3048 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 3049 3050 /* ARGH */ 3051 WARN_ON_ONCE(my_pending.stop_pending); 3052 3053 return 0; 3054 } 3055 3056 /* 3057 * Called with both p->pi_lock and rq->lock held; drops both before returning. 3058 */ 3059 static int __set_cpus_allowed_ptr_locked(struct task_struct *p, 3060 struct affinity_context *ctx, 3061 struct rq *rq, 3062 struct rq_flags *rf) 3063 __releases(rq->lock) 3064 __releases(p->pi_lock) 3065 { 3066 const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); 3067 const struct cpumask *cpu_valid_mask = cpu_active_mask; 3068 bool kthread = p->flags & PF_KTHREAD; 3069 unsigned int dest_cpu; 3070 int ret = 0; 3071 3072 update_rq_clock(rq); 3073 3074 if (kthread || is_migration_disabled(p)) { 3075 /* 3076 * Kernel threads are allowed on online && !active CPUs, 3077 * however, during cpu-hot-unplug, even these might get pushed 3078 * away if not KTHREAD_IS_PER_CPU. 3079 * 3080 * Specifically, migration_disabled() tasks must not fail the 3081 * cpumask_any_and_distribute() pick below, esp. so on 3082 * SCA_MIGRATE_ENABLE, otherwise we'll not call 3083 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 3084 */ 3085 cpu_valid_mask = cpu_online_mask; 3086 } 3087 3088 if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { 3089 ret = -EINVAL; 3090 goto out; 3091 } 3092 3093 /* 3094 * Must re-check here, to close a race against __kthread_bind(), 3095 * sched_setaffinity() is not guaranteed to observe the flag. 3096 */ 3097 if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 3098 ret = -EINVAL; 3099 goto out; 3100 } 3101 3102 if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { 3103 if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) { 3104 if (ctx->flags & SCA_USER) 3105 swap(p->user_cpus_ptr, ctx->user_mask); 3106 goto out; 3107 } 3108 3109 if (WARN_ON_ONCE(p == current && 3110 is_migration_disabled(p) && 3111 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { 3112 ret = -EBUSY; 3113 goto out; 3114 } 3115 } 3116 3117 /* 3118 * Picking a ~random cpu helps in cases where we are changing affinity 3119 * for groups of tasks (ie. cpuset), so that load balancing is not 3120 * immediately required to distribute the tasks within their new mask. 3121 */ 3122 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); 3123 if (dest_cpu >= nr_cpu_ids) { 3124 ret = -EINVAL; 3125 goto out; 3126 } 3127 3128 __do_set_cpus_allowed(p, ctx); 3129 3130 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); 3131 3132 out: 3133 task_rq_unlock(rq, p, rf); 3134 3135 return ret; 3136 } 3137 3138 /* 3139 * Change a given task's CPU affinity. Migrate the thread to a 3140 * proper CPU and schedule it away if the CPU it's executing on 3141 * is removed from the allowed bitmask. 3142 * 3143 * NOTE: the caller must have a valid reference to the task, the 3144 * task must not exit() & deallocate itself prematurely. The 3145 * call is not atomic; no spinlocks may be held. 3146 */ 3147 int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx) 3148 { 3149 struct rq_flags rf; 3150 struct rq *rq; 3151 3152 rq = task_rq_lock(p, &rf); 3153 /* 3154 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* 3155 * flags are set. 3156 */ 3157 if (p->user_cpus_ptr && 3158 !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && 3159 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) 3160 ctx->new_mask = rq->scratch_mask; 3161 3162 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); 3163 } 3164 3165 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 3166 { 3167 struct affinity_context ac = { 3168 .new_mask = new_mask, 3169 .flags = 0, 3170 }; 3171 3172 return __set_cpus_allowed_ptr(p, &ac); 3173 } 3174 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 3175 3176 /* 3177 * Change a given task's CPU affinity to the intersection of its current 3178 * affinity mask and @subset_mask, writing the resulting mask to @new_mask. 3179 * If user_cpus_ptr is defined, use it as the basis for restricting CPU 3180 * affinity or use cpu_online_mask instead. 3181 * 3182 * If the resulting mask is empty, leave the affinity unchanged and return 3183 * -EINVAL. 3184 */ 3185 static int restrict_cpus_allowed_ptr(struct task_struct *p, 3186 struct cpumask *new_mask, 3187 const struct cpumask *subset_mask) 3188 { 3189 struct affinity_context ac = { 3190 .new_mask = new_mask, 3191 .flags = 0, 3192 }; 3193 struct rq_flags rf; 3194 struct rq *rq; 3195 int err; 3196 3197 rq = task_rq_lock(p, &rf); 3198 3199 /* 3200 * Forcefully restricting the affinity of a deadline task is 3201 * likely to cause problems, so fail and noisily override the 3202 * mask entirely. 3203 */ 3204 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 3205 err = -EPERM; 3206 goto err_unlock; 3207 } 3208 3209 if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { 3210 err = -EINVAL; 3211 goto err_unlock; 3212 } 3213 3214 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); 3215 3216 err_unlock: 3217 task_rq_unlock(rq, p, &rf); 3218 return err; 3219 } 3220 3221 /* 3222 * Restrict the CPU affinity of task @p so that it is a subset of 3223 * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the 3224 * old affinity mask. If the resulting mask is empty, we warn and walk 3225 * up the cpuset hierarchy until we find a suitable mask. 3226 */ 3227 void force_compatible_cpus_allowed_ptr(struct task_struct *p) 3228 { 3229 cpumask_var_t new_mask; 3230 const struct cpumask *override_mask = task_cpu_possible_mask(p); 3231 3232 alloc_cpumask_var(&new_mask, GFP_KERNEL); 3233 3234 /* 3235 * __migrate_task() can fail silently in the face of concurrent 3236 * offlining of the chosen destination CPU, so take the hotplug 3237 * lock to ensure that the migration succeeds. 3238 */ 3239 cpus_read_lock(); 3240 if (!cpumask_available(new_mask)) 3241 goto out_set_mask; 3242 3243 if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) 3244 goto out_free_mask; 3245 3246 /* 3247 * We failed to find a valid subset of the affinity mask for the 3248 * task, so override it based on its cpuset hierarchy. 3249 */ 3250 cpuset_cpus_allowed(p, new_mask); 3251 override_mask = new_mask; 3252 3253 out_set_mask: 3254 if (printk_ratelimit()) { 3255 printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", 3256 task_pid_nr(p), p->comm, 3257 cpumask_pr_args(override_mask)); 3258 } 3259 3260 WARN_ON(set_cpus_allowed_ptr(p, override_mask)); 3261 out_free_mask: 3262 cpus_read_unlock(); 3263 free_cpumask_var(new_mask); 3264 } 3265 3266 /* 3267 * Restore the affinity of a task @p which was previously restricted by a 3268 * call to force_compatible_cpus_allowed_ptr(). 3269 * 3270 * It is the caller's responsibility to serialise this with any calls to 3271 * force_compatible_cpus_allowed_ptr(@p). 3272 */ 3273 void relax_compatible_cpus_allowed_ptr(struct task_struct *p) 3274 { 3275 struct affinity_context ac = { 3276 .new_mask = task_user_cpus(p), 3277 .flags = 0, 3278 }; 3279 int ret; 3280 3281 /* 3282 * Try to restore the old affinity mask with __sched_setaffinity(). 3283 * Cpuset masking will be done there too. 3284 */ 3285 ret = __sched_setaffinity(p, &ac); 3286 WARN_ON_ONCE(ret); 3287 } 3288 3289 #ifdef CONFIG_SMP 3290 3291 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 3292 { 3293 unsigned int state = READ_ONCE(p->__state); 3294 3295 /* 3296 * We should never call set_task_cpu() on a blocked task, 3297 * ttwu() will sort out the placement. 3298 */ 3299 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 3300 3301 /* 3302 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 3303 * because schedstat_wait_{start,end} rebase migrating task's wait_start 3304 * time relying on p->on_rq. 3305 */ 3306 WARN_ON_ONCE(state == TASK_RUNNING && 3307 p->sched_class == &fair_sched_class && 3308 (p->on_rq && !task_on_rq_migrating(p))); 3309 3310 #ifdef CONFIG_LOCKDEP 3311 /* 3312 * The caller should hold either p->pi_lock or rq->lock, when changing 3313 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 3314 * 3315 * sched_move_task() holds both and thus holding either pins the cgroup, 3316 * see task_group(). 3317 * 3318 * Furthermore, all task_rq users should acquire both locks, see 3319 * task_rq_lock(). 3320 */ 3321 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 3322 lockdep_is_held(__rq_lockp(task_rq(p))))); 3323 #endif 3324 /* 3325 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 3326 */ 3327 WARN_ON_ONCE(!cpu_online(new_cpu)); 3328 3329 WARN_ON_ONCE(is_migration_disabled(p)); 3330 3331 trace_sched_migrate_task(p, new_cpu); 3332 3333 if (task_cpu(p) != new_cpu) { 3334 if (p->sched_class->migrate_task_rq) 3335 p->sched_class->migrate_task_rq(p, new_cpu); 3336 p->se.nr_migrations++; 3337 rseq_migrate(p); 3338 sched_mm_cid_migrate_from(p); 3339 perf_event_task_migrate(p); 3340 } 3341 3342 __set_task_cpu(p, new_cpu); 3343 } 3344 #endif /* CONFIG_SMP */ 3345 3346 #ifdef CONFIG_NUMA_BALANCING 3347 static void __migrate_swap_task(struct task_struct *p, int cpu) 3348 { 3349 if (task_on_rq_queued(p)) { 3350 struct rq *src_rq, *dst_rq; 3351 struct rq_flags srf, drf; 3352 3353 src_rq = task_rq(p); 3354 dst_rq = cpu_rq(cpu); 3355 3356 rq_pin_lock(src_rq, &srf); 3357 rq_pin_lock(dst_rq, &drf); 3358 3359 move_queued_task_locked(src_rq, dst_rq, p); 3360 wakeup_preempt(dst_rq, p, 0); 3361 3362 rq_unpin_lock(dst_rq, &drf); 3363 rq_unpin_lock(src_rq, &srf); 3364 3365 } else { 3366 /* 3367 * Task isn't running anymore; make it appear like we migrated 3368 * it before it went to sleep. This means on wakeup we make the 3369 * previous CPU our target instead of where it really is. 3370 */ 3371 p->wake_cpu = cpu; 3372 } 3373 } 3374 3375 struct migration_swap_arg { 3376 struct task_struct *src_task, *dst_task; 3377 int src_cpu, dst_cpu; 3378 }; 3379 3380 static int migrate_swap_stop(void *data) 3381 { 3382 struct migration_swap_arg *arg = data; 3383 struct rq *src_rq, *dst_rq; 3384 3385 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 3386 return -EAGAIN; 3387 3388 src_rq = cpu_rq(arg->src_cpu); 3389 dst_rq = cpu_rq(arg->dst_cpu); 3390 3391 guard(double_raw_spinlock)(&arg->src_task->pi_lock, &arg->dst_task->pi_lock); 3392 guard(double_rq_lock)(src_rq, dst_rq); 3393 3394 if (task_cpu(arg->dst_task) != arg->dst_cpu) 3395 return -EAGAIN; 3396 3397 if (task_cpu(arg->src_task) != arg->src_cpu) 3398 return -EAGAIN; 3399 3400 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 3401 return -EAGAIN; 3402 3403 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 3404 return -EAGAIN; 3405 3406 __migrate_swap_task(arg->src_task, arg->dst_cpu); 3407 __migrate_swap_task(arg->dst_task, arg->src_cpu); 3408 3409 return 0; 3410 } 3411 3412 /* 3413 * Cross migrate two tasks 3414 */ 3415 int migrate_swap(struct task_struct *cur, struct task_struct *p, 3416 int target_cpu, int curr_cpu) 3417 { 3418 struct migration_swap_arg arg; 3419 int ret = -EINVAL; 3420 3421 arg = (struct migration_swap_arg){ 3422 .src_task = cur, 3423 .src_cpu = curr_cpu, 3424 .dst_task = p, 3425 .dst_cpu = target_cpu, 3426 }; 3427 3428 if (arg.src_cpu == arg.dst_cpu) 3429 goto out; 3430 3431 /* 3432 * These three tests are all lockless; this is OK since all of them 3433 * will be re-checked with proper locks held further down the line. 3434 */ 3435 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 3436 goto out; 3437 3438 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 3439 goto out; 3440 3441 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 3442 goto out; 3443 3444 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 3445 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 3446 3447 out: 3448 return ret; 3449 } 3450 #endif /* CONFIG_NUMA_BALANCING */ 3451 3452 /*** 3453 * kick_process - kick a running thread to enter/exit the kernel 3454 * @p: the to-be-kicked thread 3455 * 3456 * Cause a process which is running on another CPU to enter 3457 * kernel-mode, without any delay. (to get signals handled.) 3458 * 3459 * NOTE: this function doesn't have to take the runqueue lock, 3460 * because all it wants to ensure is that the remote task enters 3461 * the kernel. If the IPI races and the task has been migrated 3462 * to another CPU then no harm is done and the purpose has been 3463 * achieved as well. 3464 */ 3465 void kick_process(struct task_struct *p) 3466 { 3467 guard(preempt)(); 3468 int cpu = task_cpu(p); 3469 3470 if ((cpu != smp_processor_id()) && task_curr(p)) 3471 smp_send_reschedule(cpu); 3472 } 3473 EXPORT_SYMBOL_GPL(kick_process); 3474 3475 /* 3476 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3477 * 3478 * A few notes on cpu_active vs cpu_online: 3479 * 3480 * - cpu_active must be a subset of cpu_online 3481 * 3482 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3483 * see __set_cpus_allowed_ptr(). At this point the newly online 3484 * CPU isn't yet part of the sched domains, and balancing will not 3485 * see it. 3486 * 3487 * - on CPU-down we clear cpu_active() to mask the sched domains and 3488 * avoid the load balancer to place new tasks on the to be removed 3489 * CPU. Existing tasks will remain running there and will be taken 3490 * off. 3491 * 3492 * This means that fallback selection must not select !active CPUs. 3493 * And can assume that any active CPU must be online. Conversely 3494 * select_task_rq() below may allow selection of !active CPUs in order 3495 * to satisfy the above rules. 3496 */ 3497 static int select_fallback_rq(int cpu, struct task_struct *p) 3498 { 3499 int nid = cpu_to_node(cpu); 3500 const struct cpumask *nodemask = NULL; 3501 enum { cpuset, possible, fail } state = cpuset; 3502 int dest_cpu; 3503 3504 /* 3505 * If the node that the CPU is on has been offlined, cpu_to_node() 3506 * will return -1. There is no CPU on the node, and we should 3507 * select the CPU on the other node. 3508 */ 3509 if (nid != -1) { 3510 nodemask = cpumask_of_node(nid); 3511 3512 /* Look for allowed, online CPU in same node. */ 3513 for_each_cpu(dest_cpu, nodemask) { 3514 if (is_cpu_allowed(p, dest_cpu)) 3515 return dest_cpu; 3516 } 3517 } 3518 3519 for (;;) { 3520 /* Any allowed, online CPU? */ 3521 for_each_cpu(dest_cpu, p->cpus_ptr) { 3522 if (!is_cpu_allowed(p, dest_cpu)) 3523 continue; 3524 3525 goto out; 3526 } 3527 3528 /* No more Mr. Nice Guy. */ 3529 switch (state) { 3530 case cpuset: 3531 if (cpuset_cpus_allowed_fallback(p)) { 3532 state = possible; 3533 break; 3534 } 3535 fallthrough; 3536 case possible: 3537 /* 3538 * XXX When called from select_task_rq() we only 3539 * hold p->pi_lock and again violate locking order. 3540 * 3541 * More yuck to audit. 3542 */ 3543 do_set_cpus_allowed(p, task_cpu_fallback_mask(p)); 3544 state = fail; 3545 break; 3546 case fail: 3547 BUG(); 3548 break; 3549 } 3550 } 3551 3552 out: 3553 if (state != cpuset) { 3554 /* 3555 * Don't tell them about moving exiting tasks or 3556 * kernel threads (both mm NULL), since they never 3557 * leave kernel. 3558 */ 3559 if (p->mm && printk_ratelimit()) { 3560 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3561 task_pid_nr(p), p->comm, cpu); 3562 } 3563 } 3564 3565 return dest_cpu; 3566 } 3567 3568 /* 3569 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3570 */ 3571 static inline 3572 int select_task_rq(struct task_struct *p, int cpu, int *wake_flags) 3573 { 3574 lockdep_assert_held(&p->pi_lock); 3575 3576 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) { 3577 cpu = p->sched_class->select_task_rq(p, cpu, *wake_flags); 3578 *wake_flags |= WF_RQ_SELECTED; 3579 } else { 3580 cpu = cpumask_any(p->cpus_ptr); 3581 } 3582 3583 /* 3584 * In order not to call set_task_cpu() on a blocking task we need 3585 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3586 * CPU. 3587 * 3588 * Since this is common to all placement strategies, this lives here. 3589 * 3590 * [ this allows ->select_task() to simply return task_cpu(p) and 3591 * not worry about this generic constraint ] 3592 */ 3593 if (unlikely(!is_cpu_allowed(p, cpu))) 3594 cpu = select_fallback_rq(task_cpu(p), p); 3595 3596 return cpu; 3597 } 3598 3599 void sched_set_stop_task(int cpu, struct task_struct *stop) 3600 { 3601 static struct lock_class_key stop_pi_lock; 3602 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3603 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3604 3605 if (stop) { 3606 /* 3607 * Make it appear like a SCHED_FIFO task, its something 3608 * userspace knows about and won't get confused about. 3609 * 3610 * Also, it will make PI more or less work without too 3611 * much confusion -- but then, stop work should not 3612 * rely on PI working anyway. 3613 */ 3614 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3615 3616 stop->sched_class = &stop_sched_class; 3617 3618 /* 3619 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3620 * adjust the effective priority of a task. As a result, 3621 * rt_mutex_setprio() can trigger (RT) balancing operations, 3622 * which can then trigger wakeups of the stop thread to push 3623 * around the current task. 3624 * 3625 * The stop task itself will never be part of the PI-chain, it 3626 * never blocks, therefore that ->pi_lock recursion is safe. 3627 * Tell lockdep about this by placing the stop->pi_lock in its 3628 * own class. 3629 */ 3630 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3631 } 3632 3633 cpu_rq(cpu)->stop = stop; 3634 3635 if (old_stop) { 3636 /* 3637 * Reset it back to a normal scheduling class so that 3638 * it can die in pieces. 3639 */ 3640 old_stop->sched_class = &rt_sched_class; 3641 } 3642 } 3643 3644 static void 3645 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3646 { 3647 struct rq *rq; 3648 3649 if (!schedstat_enabled()) 3650 return; 3651 3652 rq = this_rq(); 3653 3654 if (cpu == rq->cpu) { 3655 __schedstat_inc(rq->ttwu_local); 3656 __schedstat_inc(p->stats.nr_wakeups_local); 3657 } else { 3658 struct sched_domain *sd; 3659 3660 __schedstat_inc(p->stats.nr_wakeups_remote); 3661 3662 guard(rcu)(); 3663 for_each_domain(rq->cpu, sd) { 3664 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3665 __schedstat_inc(sd->ttwu_wake_remote); 3666 break; 3667 } 3668 } 3669 } 3670 3671 if (wake_flags & WF_MIGRATED) 3672 __schedstat_inc(p->stats.nr_wakeups_migrate); 3673 3674 __schedstat_inc(rq->ttwu_count); 3675 __schedstat_inc(p->stats.nr_wakeups); 3676 3677 if (wake_flags & WF_SYNC) 3678 __schedstat_inc(p->stats.nr_wakeups_sync); 3679 } 3680 3681 /* 3682 * Mark the task runnable. 3683 */ 3684 static inline void ttwu_do_wakeup(struct task_struct *p) 3685 { 3686 WRITE_ONCE(p->__state, TASK_RUNNING); 3687 trace_sched_wakeup(p); 3688 } 3689 3690 static void 3691 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3692 struct rq_flags *rf) 3693 { 3694 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3695 3696 lockdep_assert_rq_held(rq); 3697 3698 if (p->sched_contributes_to_load) 3699 rq->nr_uninterruptible--; 3700 3701 if (wake_flags & WF_RQ_SELECTED) 3702 en_flags |= ENQUEUE_RQ_SELECTED; 3703 if (wake_flags & WF_MIGRATED) 3704 en_flags |= ENQUEUE_MIGRATED; 3705 else 3706 if (p->in_iowait) { 3707 delayacct_blkio_end(p); 3708 atomic_dec(&task_rq(p)->nr_iowait); 3709 } 3710 3711 activate_task(rq, p, en_flags); 3712 wakeup_preempt(rq, p, wake_flags); 3713 3714 ttwu_do_wakeup(p); 3715 3716 if (p->sched_class->task_woken) { 3717 /* 3718 * Our task @p is fully woken up and running; so it's safe to 3719 * drop the rq->lock, hereafter rq is only used for statistics. 3720 */ 3721 rq_unpin_lock(rq, rf); 3722 p->sched_class->task_woken(rq, p); 3723 rq_repin_lock(rq, rf); 3724 } 3725 3726 if (rq->idle_stamp) { 3727 u64 delta = rq_clock(rq) - rq->idle_stamp; 3728 u64 max = 2*rq->max_idle_balance_cost; 3729 3730 update_avg(&rq->avg_idle, delta); 3731 3732 if (rq->avg_idle > max) 3733 rq->avg_idle = max; 3734 3735 rq->idle_stamp = 0; 3736 } 3737 } 3738 3739 /* 3740 * Consider @p being inside a wait loop: 3741 * 3742 * for (;;) { 3743 * set_current_state(TASK_UNINTERRUPTIBLE); 3744 * 3745 * if (CONDITION) 3746 * break; 3747 * 3748 * schedule(); 3749 * } 3750 * __set_current_state(TASK_RUNNING); 3751 * 3752 * between set_current_state() and schedule(). In this case @p is still 3753 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3754 * an atomic manner. 3755 * 3756 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3757 * then schedule() must still happen and p->state can be changed to 3758 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3759 * need to do a full wakeup with enqueue. 3760 * 3761 * Returns: %true when the wakeup is done, 3762 * %false otherwise. 3763 */ 3764 static int ttwu_runnable(struct task_struct *p, int wake_flags) 3765 { 3766 struct rq_flags rf; 3767 struct rq *rq; 3768 int ret = 0; 3769 3770 rq = __task_rq_lock(p, &rf); 3771 if (task_on_rq_queued(p)) { 3772 update_rq_clock(rq); 3773 if (p->se.sched_delayed) 3774 enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); 3775 if (!task_on_cpu(rq, p)) { 3776 /* 3777 * When on_rq && !on_cpu the task is preempted, see if 3778 * it should preempt the task that is current now. 3779 */ 3780 wakeup_preempt(rq, p, wake_flags); 3781 } 3782 ttwu_do_wakeup(p); 3783 ret = 1; 3784 } 3785 __task_rq_unlock(rq, &rf); 3786 3787 return ret; 3788 } 3789 3790 void sched_ttwu_pending(void *arg) 3791 { 3792 struct llist_node *llist = arg; 3793 struct rq *rq = this_rq(); 3794 struct task_struct *p, *t; 3795 struct rq_flags rf; 3796 3797 if (!llist) 3798 return; 3799 3800 rq_lock_irqsave(rq, &rf); 3801 update_rq_clock(rq); 3802 3803 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3804 if (WARN_ON_ONCE(p->on_cpu)) 3805 smp_cond_load_acquire(&p->on_cpu, !VAL); 3806 3807 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3808 set_task_cpu(p, cpu_of(rq)); 3809 3810 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3811 } 3812 3813 /* 3814 * Must be after enqueueing at least once task such that 3815 * idle_cpu() does not observe a false-negative -- if it does, 3816 * it is possible for select_idle_siblings() to stack a number 3817 * of tasks on this CPU during that window. 3818 * 3819 * It is OK to clear ttwu_pending when another task pending. 3820 * We will receive IPI after local IRQ enabled and then enqueue it. 3821 * Since now nr_running > 0, idle_cpu() will always get correct result. 3822 */ 3823 WRITE_ONCE(rq->ttwu_pending, 0); 3824 rq_unlock_irqrestore(rq, &rf); 3825 } 3826 3827 /* 3828 * Prepare the scene for sending an IPI for a remote smp_call 3829 * 3830 * Returns true if the caller can proceed with sending the IPI. 3831 * Returns false otherwise. 3832 */ 3833 bool call_function_single_prep_ipi(int cpu) 3834 { 3835 if (set_nr_if_polling(cpu_rq(cpu)->idle)) { 3836 trace_sched_wake_idle_without_ipi(cpu); 3837 return false; 3838 } 3839 3840 return true; 3841 } 3842 3843 /* 3844 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3845 * necessary. The wakee CPU on receipt of the IPI will queue the task 3846 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3847 * of the wakeup instead of the waker. 3848 */ 3849 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3850 { 3851 struct rq *rq = cpu_rq(cpu); 3852 3853 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3854 3855 WRITE_ONCE(rq->ttwu_pending, 1); 3856 #ifdef CONFIG_SMP 3857 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3858 #endif 3859 } 3860 3861 void wake_up_if_idle(int cpu) 3862 { 3863 struct rq *rq = cpu_rq(cpu); 3864 3865 guard(rcu)(); 3866 if (is_idle_task(rcu_dereference(rq->curr))) { 3867 guard(rq_lock_irqsave)(rq); 3868 if (is_idle_task(rq->curr)) 3869 resched_curr(rq); 3870 } 3871 } 3872 3873 bool cpus_equal_capacity(int this_cpu, int that_cpu) 3874 { 3875 if (!sched_asym_cpucap_active()) 3876 return true; 3877 3878 if (this_cpu == that_cpu) 3879 return true; 3880 3881 return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu); 3882 } 3883 3884 bool cpus_share_cache(int this_cpu, int that_cpu) 3885 { 3886 if (this_cpu == that_cpu) 3887 return true; 3888 3889 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3890 } 3891 3892 /* 3893 * Whether CPUs are share cache resources, which means LLC on non-cluster 3894 * machines and LLC tag or L2 on machines with clusters. 3895 */ 3896 bool cpus_share_resources(int this_cpu, int that_cpu) 3897 { 3898 if (this_cpu == that_cpu) 3899 return true; 3900 3901 return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu); 3902 } 3903 3904 static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) 3905 { 3906 /* See SCX_OPS_ALLOW_QUEUED_WAKEUP. */ 3907 if (!scx_allow_ttwu_queue(p)) 3908 return false; 3909 3910 #ifdef CONFIG_SMP 3911 if (p->sched_class == &stop_sched_class) 3912 return false; 3913 #endif 3914 3915 /* 3916 * Do not complicate things with the async wake_list while the CPU is 3917 * in hotplug state. 3918 */ 3919 if (!cpu_active(cpu)) 3920 return false; 3921 3922 /* Ensure the task will still be allowed to run on the CPU. */ 3923 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 3924 return false; 3925 3926 /* 3927 * If the CPU does not share cache, then queue the task on the 3928 * remote rqs wakelist to avoid accessing remote data. 3929 */ 3930 if (!cpus_share_cache(smp_processor_id(), cpu)) 3931 return true; 3932 3933 if (cpu == smp_processor_id()) 3934 return false; 3935 3936 /* 3937 * If the wakee cpu is idle, or the task is descheduling and the 3938 * only running task on the CPU, then use the wakelist to offload 3939 * the task activation to the idle (or soon-to-be-idle) CPU as 3940 * the current CPU is likely busy. nr_running is checked to 3941 * avoid unnecessary task stacking. 3942 * 3943 * Note that we can only get here with (wakee) p->on_rq=0, 3944 * p->on_cpu can be whatever, we've done the dequeue, so 3945 * the wakee has been accounted out of ->nr_running. 3946 */ 3947 if (!cpu_rq(cpu)->nr_running) 3948 return true; 3949 3950 return false; 3951 } 3952 3953 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3954 { 3955 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { 3956 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 3957 __ttwu_queue_wakelist(p, cpu, wake_flags); 3958 return true; 3959 } 3960 3961 return false; 3962 } 3963 3964 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 3965 { 3966 struct rq *rq = cpu_rq(cpu); 3967 struct rq_flags rf; 3968 3969 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 3970 return; 3971 3972 rq_lock(rq, &rf); 3973 update_rq_clock(rq); 3974 ttwu_do_activate(rq, p, wake_flags, &rf); 3975 rq_unlock(rq, &rf); 3976 } 3977 3978 /* 3979 * Invoked from try_to_wake_up() to check whether the task can be woken up. 3980 * 3981 * The caller holds p::pi_lock if p != current or has preemption 3982 * disabled when p == current. 3983 * 3984 * The rules of saved_state: 3985 * 3986 * The related locking code always holds p::pi_lock when updating 3987 * p::saved_state, which means the code is fully serialized in both cases. 3988 * 3989 * For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. 3990 * No other bits set. This allows to distinguish all wakeup scenarios. 3991 * 3992 * For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This 3993 * allows us to prevent early wakeup of tasks before they can be run on 3994 * asymmetric ISA architectures (eg ARMv9). 3995 */ 3996 static __always_inline 3997 bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 3998 { 3999 int match; 4000 4001 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 4002 WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && 4003 state != TASK_RTLOCK_WAIT); 4004 } 4005 4006 *success = !!(match = __task_state_match(p, state)); 4007 4008 /* 4009 * Saved state preserves the task state across blocking on 4010 * an RT lock or TASK_FREEZABLE tasks. If the state matches, 4011 * set p::saved_state to TASK_RUNNING, but do not wake the task 4012 * because it waits for a lock wakeup or __thaw_task(). Also 4013 * indicate success because from the regular waker's point of 4014 * view this has succeeded. 4015 * 4016 * After acquiring the lock the task will restore p::__state 4017 * from p::saved_state which ensures that the regular 4018 * wakeup is not lost. The restore will also set 4019 * p::saved_state to TASK_RUNNING so any further tests will 4020 * not result in false positives vs. @success 4021 */ 4022 if (match < 0) 4023 p->saved_state = TASK_RUNNING; 4024 4025 return match > 0; 4026 } 4027 4028 /* 4029 * Notes on Program-Order guarantees on SMP systems. 4030 * 4031 * MIGRATION 4032 * 4033 * The basic program-order guarantee on SMP systems is that when a task [t] 4034 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 4035 * execution on its new CPU [c1]. 4036 * 4037 * For migration (of runnable tasks) this is provided by the following means: 4038 * 4039 * A) UNLOCK of the rq(c0)->lock scheduling out task t 4040 * B) migration for t is required to synchronize *both* rq(c0)->lock and 4041 * rq(c1)->lock (if not at the same time, then in that order). 4042 * C) LOCK of the rq(c1)->lock scheduling in task 4043 * 4044 * Release/acquire chaining guarantees that B happens after A and C after B. 4045 * Note: the CPU doing B need not be c0 or c1 4046 * 4047 * Example: 4048 * 4049 * CPU0 CPU1 CPU2 4050 * 4051 * LOCK rq(0)->lock 4052 * sched-out X 4053 * sched-in Y 4054 * UNLOCK rq(0)->lock 4055 * 4056 * LOCK rq(0)->lock // orders against CPU0 4057 * dequeue X 4058 * UNLOCK rq(0)->lock 4059 * 4060 * LOCK rq(1)->lock 4061 * enqueue X 4062 * UNLOCK rq(1)->lock 4063 * 4064 * LOCK rq(1)->lock // orders against CPU2 4065 * sched-out Z 4066 * sched-in X 4067 * UNLOCK rq(1)->lock 4068 * 4069 * 4070 * BLOCKING -- aka. SLEEP + WAKEUP 4071 * 4072 * For blocking we (obviously) need to provide the same guarantee as for 4073 * migration. However the means are completely different as there is no lock 4074 * chain to provide order. Instead we do: 4075 * 4076 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 4077 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 4078 * 4079 * Example: 4080 * 4081 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 4082 * 4083 * LOCK rq(0)->lock LOCK X->pi_lock 4084 * dequeue X 4085 * sched-out X 4086 * smp_store_release(X->on_cpu, 0); 4087 * 4088 * smp_cond_load_acquire(&X->on_cpu, !VAL); 4089 * X->state = WAKING 4090 * set_task_cpu(X,2) 4091 * 4092 * LOCK rq(2)->lock 4093 * enqueue X 4094 * X->state = RUNNING 4095 * UNLOCK rq(2)->lock 4096 * 4097 * LOCK rq(2)->lock // orders against CPU1 4098 * sched-out Z 4099 * sched-in X 4100 * UNLOCK rq(2)->lock 4101 * 4102 * UNLOCK X->pi_lock 4103 * UNLOCK rq(0)->lock 4104 * 4105 * 4106 * However, for wakeups there is a second guarantee we must provide, namely we 4107 * must ensure that CONDITION=1 done by the caller can not be reordered with 4108 * accesses to the task state; see try_to_wake_up() and set_current_state(). 4109 */ 4110 4111 /** 4112 * try_to_wake_up - wake up a thread 4113 * @p: the thread to be awakened 4114 * @state: the mask of task states that can be woken 4115 * @wake_flags: wake modifier flags (WF_*) 4116 * 4117 * Conceptually does: 4118 * 4119 * If (@state & @p->state) @p->state = TASK_RUNNING. 4120 * 4121 * If the task was not queued/runnable, also place it back on a runqueue. 4122 * 4123 * This function is atomic against schedule() which would dequeue the task. 4124 * 4125 * It issues a full memory barrier before accessing @p->state, see the comment 4126 * with set_current_state(). 4127 * 4128 * Uses p->pi_lock to serialize against concurrent wake-ups. 4129 * 4130 * Relies on p->pi_lock stabilizing: 4131 * - p->sched_class 4132 * - p->cpus_ptr 4133 * - p->sched_task_group 4134 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 4135 * 4136 * Tries really hard to only take one task_rq(p)->lock for performance. 4137 * Takes rq->lock in: 4138 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 4139 * - ttwu_queue() -- new rq, for enqueue of the task; 4140 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 4141 * 4142 * As a consequence we race really badly with just about everything. See the 4143 * many memory barriers and their comments for details. 4144 * 4145 * Return: %true if @p->state changes (an actual wakeup was done), 4146 * %false otherwise. 4147 */ 4148 int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 4149 { 4150 guard(preempt)(); 4151 int cpu, success = 0; 4152 4153 wake_flags |= WF_TTWU; 4154 4155 if (p == current) { 4156 /* 4157 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 4158 * == smp_processor_id()'. Together this means we can special 4159 * case the whole 'p->on_rq && ttwu_runnable()' case below 4160 * without taking any locks. 4161 * 4162 * Specifically, given current runs ttwu() we must be before 4163 * schedule()'s block_task(), as such this must not observe 4164 * sched_delayed. 4165 * 4166 * In particular: 4167 * - we rely on Program-Order guarantees for all the ordering, 4168 * - we're serialized against set_special_state() by virtue of 4169 * it disabling IRQs (this allows not taking ->pi_lock). 4170 */ 4171 WARN_ON_ONCE(p->se.sched_delayed); 4172 if (!ttwu_state_match(p, state, &success)) 4173 goto out; 4174 4175 trace_sched_waking(p); 4176 ttwu_do_wakeup(p); 4177 goto out; 4178 } 4179 4180 /* 4181 * If we are going to wake up a thread waiting for CONDITION we 4182 * need to ensure that CONDITION=1 done by the caller can not be 4183 * reordered with p->state check below. This pairs with smp_store_mb() 4184 * in set_current_state() that the waiting thread does. 4185 */ 4186 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 4187 smp_mb__after_spinlock(); 4188 if (!ttwu_state_match(p, state, &success)) 4189 break; 4190 4191 trace_sched_waking(p); 4192 4193 /* 4194 * Ensure we load p->on_rq _after_ p->state, otherwise it would 4195 * be possible to, falsely, observe p->on_rq == 0 and get stuck 4196 * in smp_cond_load_acquire() below. 4197 * 4198 * sched_ttwu_pending() try_to_wake_up() 4199 * STORE p->on_rq = 1 LOAD p->state 4200 * UNLOCK rq->lock 4201 * 4202 * __schedule() (switch to task 'p') 4203 * LOCK rq->lock smp_rmb(); 4204 * smp_mb__after_spinlock(); 4205 * UNLOCK rq->lock 4206 * 4207 * [task p] 4208 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 4209 * 4210 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4211 * __schedule(). See the comment for smp_mb__after_spinlock(). 4212 * 4213 * A similar smp_rmb() lives in __task_needs_rq_lock(). 4214 */ 4215 smp_rmb(); 4216 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 4217 break; 4218 4219 /* 4220 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 4221 * possible to, falsely, observe p->on_cpu == 0. 4222 * 4223 * One must be running (->on_cpu == 1) in order to remove oneself 4224 * from the runqueue. 4225 * 4226 * __schedule() (switch to task 'p') try_to_wake_up() 4227 * STORE p->on_cpu = 1 LOAD p->on_rq 4228 * UNLOCK rq->lock 4229 * 4230 * __schedule() (put 'p' to sleep) 4231 * LOCK rq->lock smp_rmb(); 4232 * smp_mb__after_spinlock(); 4233 * STORE p->on_rq = 0 LOAD p->on_cpu 4234 * 4235 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4236 * __schedule(). See the comment for smp_mb__after_spinlock(). 4237 * 4238 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 4239 * schedule()'s deactivate_task() has 'happened' and p will no longer 4240 * care about it's own p->state. See the comment in __schedule(). 4241 */ 4242 smp_acquire__after_ctrl_dep(); 4243 4244 /* 4245 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 4246 * == 0), which means we need to do an enqueue, change p->state to 4247 * TASK_WAKING such that we can unlock p->pi_lock before doing the 4248 * enqueue, such as ttwu_queue_wakelist(). 4249 */ 4250 WRITE_ONCE(p->__state, TASK_WAKING); 4251 4252 /* 4253 * If the owning (remote) CPU is still in the middle of schedule() with 4254 * this task as prev, considering queueing p on the remote CPUs wake_list 4255 * which potentially sends an IPI instead of spinning on p->on_cpu to 4256 * let the waker make forward progress. This is safe because IRQs are 4257 * disabled and the IPI will deliver after on_cpu is cleared. 4258 * 4259 * Ensure we load task_cpu(p) after p->on_cpu: 4260 * 4261 * set_task_cpu(p, cpu); 4262 * STORE p->cpu = @cpu 4263 * __schedule() (switch to task 'p') 4264 * LOCK rq->lock 4265 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 4266 * STORE p->on_cpu = 1 LOAD p->cpu 4267 * 4268 * to ensure we observe the correct CPU on which the task is currently 4269 * scheduling. 4270 */ 4271 if (smp_load_acquire(&p->on_cpu) && 4272 ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) 4273 break; 4274 4275 /* 4276 * If the owning (remote) CPU is still in the middle of schedule() with 4277 * this task as prev, wait until it's done referencing the task. 4278 * 4279 * Pairs with the smp_store_release() in finish_task(). 4280 * 4281 * This ensures that tasks getting woken will be fully ordered against 4282 * their previous state and preserve Program Order. 4283 */ 4284 smp_cond_load_acquire(&p->on_cpu, !VAL); 4285 4286 cpu = select_task_rq(p, p->wake_cpu, &wake_flags); 4287 if (task_cpu(p) != cpu) { 4288 if (p->in_iowait) { 4289 delayacct_blkio_end(p); 4290 atomic_dec(&task_rq(p)->nr_iowait); 4291 } 4292 4293 wake_flags |= WF_MIGRATED; 4294 psi_ttwu_dequeue(p); 4295 set_task_cpu(p, cpu); 4296 } 4297 4298 ttwu_queue(p, cpu, wake_flags); 4299 } 4300 out: 4301 if (success) 4302 ttwu_stat(p, task_cpu(p), wake_flags); 4303 4304 return success; 4305 } 4306 4307 static bool __task_needs_rq_lock(struct task_struct *p) 4308 { 4309 unsigned int state = READ_ONCE(p->__state); 4310 4311 /* 4312 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when 4313 * the task is blocked. Make sure to check @state since ttwu() can drop 4314 * locks at the end, see ttwu_queue_wakelist(). 4315 */ 4316 if (state == TASK_RUNNING || state == TASK_WAKING) 4317 return true; 4318 4319 /* 4320 * Ensure we load p->on_rq after p->__state, otherwise it would be 4321 * possible to, falsely, observe p->on_rq == 0. 4322 * 4323 * See try_to_wake_up() for a longer comment. 4324 */ 4325 smp_rmb(); 4326 if (p->on_rq) 4327 return true; 4328 4329 /* 4330 * Ensure the task has finished __schedule() and will not be referenced 4331 * anymore. Again, see try_to_wake_up() for a longer comment. 4332 */ 4333 smp_rmb(); 4334 smp_cond_load_acquire(&p->on_cpu, !VAL); 4335 4336 return false; 4337 } 4338 4339 /** 4340 * task_call_func - Invoke a function on task in fixed state 4341 * @p: Process for which the function is to be invoked, can be @current. 4342 * @func: Function to invoke. 4343 * @arg: Argument to function. 4344 * 4345 * Fix the task in it's current state by avoiding wakeups and or rq operations 4346 * and call @func(@arg) on it. This function can use task_is_runnable() and 4347 * task_curr() to work out what the state is, if required. Given that @func 4348 * can be invoked with a runqueue lock held, it had better be quite 4349 * lightweight. 4350 * 4351 * Returns: 4352 * Whatever @func returns 4353 */ 4354 int task_call_func(struct task_struct *p, task_call_f func, void *arg) 4355 { 4356 struct rq *rq = NULL; 4357 struct rq_flags rf; 4358 int ret; 4359 4360 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4361 4362 if (__task_needs_rq_lock(p)) 4363 rq = __task_rq_lock(p, &rf); 4364 4365 /* 4366 * At this point the task is pinned; either: 4367 * - blocked and we're holding off wakeups (pi->lock) 4368 * - woken, and we're holding off enqueue (rq->lock) 4369 * - queued, and we're holding off schedule (rq->lock) 4370 * - running, and we're holding off de-schedule (rq->lock) 4371 * 4372 * The called function (@func) can use: task_curr(), p->on_rq and 4373 * p->__state to differentiate between these states. 4374 */ 4375 ret = func(p, arg); 4376 4377 if (rq) 4378 rq_unlock(rq, &rf); 4379 4380 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 4381 return ret; 4382 } 4383 4384 /** 4385 * cpu_curr_snapshot - Return a snapshot of the currently running task 4386 * @cpu: The CPU on which to snapshot the task. 4387 * 4388 * Returns the task_struct pointer of the task "currently" running on 4389 * the specified CPU. 4390 * 4391 * If the specified CPU was offline, the return value is whatever it 4392 * is, perhaps a pointer to the task_struct structure of that CPU's idle 4393 * task, but there is no guarantee. Callers wishing a useful return 4394 * value must take some action to ensure that the specified CPU remains 4395 * online throughout. 4396 * 4397 * This function executes full memory barriers before and after fetching 4398 * the pointer, which permits the caller to confine this function's fetch 4399 * with respect to the caller's accesses to other shared variables. 4400 */ 4401 struct task_struct *cpu_curr_snapshot(int cpu) 4402 { 4403 struct rq *rq = cpu_rq(cpu); 4404 struct task_struct *t; 4405 struct rq_flags rf; 4406 4407 rq_lock_irqsave(rq, &rf); 4408 smp_mb__after_spinlock(); /* Pairing determined by caller's synchronization design. */ 4409 t = rcu_dereference(cpu_curr(cpu)); 4410 rq_unlock_irqrestore(rq, &rf); 4411 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4412 4413 return t; 4414 } 4415 4416 /** 4417 * wake_up_process - Wake up a specific process 4418 * @p: The process to be woken up. 4419 * 4420 * Attempt to wake up the nominated process and move it to the set of runnable 4421 * processes. 4422 * 4423 * Return: 1 if the process was woken up, 0 if it was already running. 4424 * 4425 * This function executes a full memory barrier before accessing the task state. 4426 */ 4427 int wake_up_process(struct task_struct *p) 4428 { 4429 return try_to_wake_up(p, TASK_NORMAL, 0); 4430 } 4431 EXPORT_SYMBOL(wake_up_process); 4432 4433 int wake_up_state(struct task_struct *p, unsigned int state) 4434 { 4435 return try_to_wake_up(p, state, 0); 4436 } 4437 4438 /* 4439 * Perform scheduler related setup for a newly forked process p. 4440 * p is forked by current. 4441 * 4442 * __sched_fork() is basic setup which is also used by sched_init() to 4443 * initialize the boot CPU's idle task. 4444 */ 4445 static void __sched_fork(u64 clone_flags, struct task_struct *p) 4446 { 4447 p->on_rq = 0; 4448 4449 p->se.on_rq = 0; 4450 p->se.exec_start = 0; 4451 p->se.sum_exec_runtime = 0; 4452 p->se.prev_sum_exec_runtime = 0; 4453 p->se.nr_migrations = 0; 4454 p->se.vruntime = 0; 4455 p->se.vlag = 0; 4456 p->se.rel_deadline = 0; 4457 INIT_LIST_HEAD(&p->se.group_node); 4458 4459 /* A delayed task cannot be in clone(). */ 4460 WARN_ON_ONCE(p->se.sched_delayed); 4461 4462 #ifdef CONFIG_FAIR_GROUP_SCHED 4463 p->se.cfs_rq = NULL; 4464 #ifdef CONFIG_CFS_BANDWIDTH 4465 init_cfs_throttle_work(p); 4466 #endif 4467 #endif 4468 4469 #ifdef CONFIG_SCHEDSTATS 4470 /* Even if schedstat is disabled, there should not be garbage */ 4471 memset(&p->stats, 0, sizeof(p->stats)); 4472 #endif 4473 4474 init_dl_entity(&p->dl); 4475 4476 INIT_LIST_HEAD(&p->rt.run_list); 4477 p->rt.timeout = 0; 4478 p->rt.time_slice = sched_rr_timeslice; 4479 p->rt.on_rq = 0; 4480 p->rt.on_list = 0; 4481 4482 #ifdef CONFIG_SCHED_CLASS_EXT 4483 init_scx_entity(&p->scx); 4484 #endif 4485 4486 #ifdef CONFIG_PREEMPT_NOTIFIERS 4487 INIT_HLIST_HEAD(&p->preempt_notifiers); 4488 #endif 4489 4490 #ifdef CONFIG_COMPACTION 4491 p->capture_control = NULL; 4492 #endif 4493 init_numa_balancing(clone_flags, p); 4494 p->wake_entry.u_flags = CSD_TYPE_TTWU; 4495 p->migration_pending = NULL; 4496 init_sched_mm_cid(p); 4497 } 4498 4499 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 4500 4501 #ifdef CONFIG_NUMA_BALANCING 4502 4503 int sysctl_numa_balancing_mode; 4504 4505 static void __set_numabalancing_state(bool enabled) 4506 { 4507 if (enabled) 4508 static_branch_enable(&sched_numa_balancing); 4509 else 4510 static_branch_disable(&sched_numa_balancing); 4511 } 4512 4513 void set_numabalancing_state(bool enabled) 4514 { 4515 if (enabled) 4516 sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; 4517 else 4518 sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; 4519 __set_numabalancing_state(enabled); 4520 } 4521 4522 #ifdef CONFIG_PROC_SYSCTL 4523 static void reset_memory_tiering(void) 4524 { 4525 struct pglist_data *pgdat; 4526 4527 for_each_online_pgdat(pgdat) { 4528 pgdat->nbp_threshold = 0; 4529 pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 4530 pgdat->nbp_th_start = jiffies_to_msecs(jiffies); 4531 } 4532 } 4533 4534 static int sysctl_numa_balancing(const struct ctl_table *table, int write, 4535 void *buffer, size_t *lenp, loff_t *ppos) 4536 { 4537 struct ctl_table t; 4538 int err; 4539 int state = sysctl_numa_balancing_mode; 4540 4541 if (write && !capable(CAP_SYS_ADMIN)) 4542 return -EPERM; 4543 4544 t = *table; 4545 t.data = &state; 4546 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4547 if (err < 0) 4548 return err; 4549 if (write) { 4550 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 4551 (state & NUMA_BALANCING_MEMORY_TIERING)) 4552 reset_memory_tiering(); 4553 sysctl_numa_balancing_mode = state; 4554 __set_numabalancing_state(state); 4555 } 4556 return err; 4557 } 4558 #endif /* CONFIG_PROC_SYSCTL */ 4559 #endif /* CONFIG_NUMA_BALANCING */ 4560 4561 #ifdef CONFIG_SCHEDSTATS 4562 4563 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4564 4565 static void set_schedstats(bool enabled) 4566 { 4567 if (enabled) 4568 static_branch_enable(&sched_schedstats); 4569 else 4570 static_branch_disable(&sched_schedstats); 4571 } 4572 4573 void force_schedstat_enabled(void) 4574 { 4575 if (!schedstat_enabled()) { 4576 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4577 static_branch_enable(&sched_schedstats); 4578 } 4579 } 4580 4581 static int __init setup_schedstats(char *str) 4582 { 4583 int ret = 0; 4584 if (!str) 4585 goto out; 4586 4587 if (!strcmp(str, "enable")) { 4588 set_schedstats(true); 4589 ret = 1; 4590 } else if (!strcmp(str, "disable")) { 4591 set_schedstats(false); 4592 ret = 1; 4593 } 4594 out: 4595 if (!ret) 4596 pr_warn("Unable to parse schedstats=\n"); 4597 4598 return ret; 4599 } 4600 __setup("schedstats=", setup_schedstats); 4601 4602 #ifdef CONFIG_PROC_SYSCTL 4603 static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer, 4604 size_t *lenp, loff_t *ppos) 4605 { 4606 struct ctl_table t; 4607 int err; 4608 int state = static_branch_likely(&sched_schedstats); 4609 4610 if (write && !capable(CAP_SYS_ADMIN)) 4611 return -EPERM; 4612 4613 t = *table; 4614 t.data = &state; 4615 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4616 if (err < 0) 4617 return err; 4618 if (write) 4619 set_schedstats(state); 4620 return err; 4621 } 4622 #endif /* CONFIG_PROC_SYSCTL */ 4623 #endif /* CONFIG_SCHEDSTATS */ 4624 4625 #ifdef CONFIG_SYSCTL 4626 static const struct ctl_table sched_core_sysctls[] = { 4627 #ifdef CONFIG_SCHEDSTATS 4628 { 4629 .procname = "sched_schedstats", 4630 .data = NULL, 4631 .maxlen = sizeof(unsigned int), 4632 .mode = 0644, 4633 .proc_handler = sysctl_schedstats, 4634 .extra1 = SYSCTL_ZERO, 4635 .extra2 = SYSCTL_ONE, 4636 }, 4637 #endif /* CONFIG_SCHEDSTATS */ 4638 #ifdef CONFIG_UCLAMP_TASK 4639 { 4640 .procname = "sched_util_clamp_min", 4641 .data = &sysctl_sched_uclamp_util_min, 4642 .maxlen = sizeof(unsigned int), 4643 .mode = 0644, 4644 .proc_handler = sysctl_sched_uclamp_handler, 4645 }, 4646 { 4647 .procname = "sched_util_clamp_max", 4648 .data = &sysctl_sched_uclamp_util_max, 4649 .maxlen = sizeof(unsigned int), 4650 .mode = 0644, 4651 .proc_handler = sysctl_sched_uclamp_handler, 4652 }, 4653 { 4654 .procname = "sched_util_clamp_min_rt_default", 4655 .data = &sysctl_sched_uclamp_util_min_rt_default, 4656 .maxlen = sizeof(unsigned int), 4657 .mode = 0644, 4658 .proc_handler = sysctl_sched_uclamp_handler, 4659 }, 4660 #endif /* CONFIG_UCLAMP_TASK */ 4661 #ifdef CONFIG_NUMA_BALANCING 4662 { 4663 .procname = "numa_balancing", 4664 .data = NULL, /* filled in by handler */ 4665 .maxlen = sizeof(unsigned int), 4666 .mode = 0644, 4667 .proc_handler = sysctl_numa_balancing, 4668 .extra1 = SYSCTL_ZERO, 4669 .extra2 = SYSCTL_FOUR, 4670 }, 4671 #endif /* CONFIG_NUMA_BALANCING */ 4672 }; 4673 static int __init sched_core_sysctl_init(void) 4674 { 4675 register_sysctl_init("kernel", sched_core_sysctls); 4676 return 0; 4677 } 4678 late_initcall(sched_core_sysctl_init); 4679 #endif /* CONFIG_SYSCTL */ 4680 4681 /* 4682 * fork()/clone()-time setup: 4683 */ 4684 int sched_fork(u64 clone_flags, struct task_struct *p) 4685 { 4686 __sched_fork(clone_flags, p); 4687 /* 4688 * We mark the process as NEW here. This guarantees that 4689 * nobody will actually run it, and a signal or other external 4690 * event cannot wake it up and insert it on the runqueue either. 4691 */ 4692 p->__state = TASK_NEW; 4693 4694 /* 4695 * Make sure we do not leak PI boosting priority to the child. 4696 */ 4697 p->prio = current->normal_prio; 4698 4699 uclamp_fork(p); 4700 4701 /* 4702 * Revert to default priority/policy on fork if requested. 4703 */ 4704 if (unlikely(p->sched_reset_on_fork)) { 4705 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4706 p->policy = SCHED_NORMAL; 4707 p->static_prio = NICE_TO_PRIO(0); 4708 p->rt_priority = 0; 4709 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4710 p->static_prio = NICE_TO_PRIO(0); 4711 4712 p->prio = p->normal_prio = p->static_prio; 4713 set_load_weight(p, false); 4714 p->se.custom_slice = 0; 4715 p->se.slice = sysctl_sched_base_slice; 4716 4717 /* 4718 * We don't need the reset flag anymore after the fork. It has 4719 * fulfilled its duty: 4720 */ 4721 p->sched_reset_on_fork = 0; 4722 } 4723 4724 if (dl_prio(p->prio)) 4725 return -EAGAIN; 4726 4727 scx_pre_fork(p); 4728 4729 if (rt_prio(p->prio)) { 4730 p->sched_class = &rt_sched_class; 4731 #ifdef CONFIG_SCHED_CLASS_EXT 4732 } else if (task_should_scx(p->policy)) { 4733 p->sched_class = &ext_sched_class; 4734 #endif 4735 } else { 4736 p->sched_class = &fair_sched_class; 4737 } 4738 4739 init_entity_runnable_average(&p->se); 4740 4741 4742 #ifdef CONFIG_SCHED_INFO 4743 if (likely(sched_info_on())) 4744 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4745 #endif 4746 p->on_cpu = 0; 4747 init_task_preempt_count(p); 4748 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4749 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4750 4751 return 0; 4752 } 4753 4754 int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4755 { 4756 unsigned long flags; 4757 4758 /* 4759 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly 4760 * required yet, but lockdep gets upset if rules are violated. 4761 */ 4762 raw_spin_lock_irqsave(&p->pi_lock, flags); 4763 #ifdef CONFIG_CGROUP_SCHED 4764 if (1) { 4765 struct task_group *tg; 4766 tg = container_of(kargs->cset->subsys[cpu_cgrp_id], 4767 struct task_group, css); 4768 tg = autogroup_task_group(p, tg); 4769 p->sched_task_group = tg; 4770 } 4771 #endif 4772 rseq_migrate(p); 4773 /* 4774 * We're setting the CPU for the first time, we don't migrate, 4775 * so use __set_task_cpu(). 4776 */ 4777 __set_task_cpu(p, smp_processor_id()); 4778 if (p->sched_class->task_fork) 4779 p->sched_class->task_fork(p); 4780 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4781 4782 return scx_fork(p); 4783 } 4784 4785 void sched_cancel_fork(struct task_struct *p) 4786 { 4787 scx_cancel_fork(p); 4788 } 4789 4790 void sched_post_fork(struct task_struct *p) 4791 { 4792 uclamp_post_fork(p); 4793 scx_post_fork(p); 4794 } 4795 4796 u64 to_ratio(u64 period, u64 runtime) 4797 { 4798 if (runtime == RUNTIME_INF) 4799 return BW_UNIT; 4800 4801 /* 4802 * Doing this here saves a lot of checks in all 4803 * the calling paths, and returning zero seems 4804 * safe for them anyway. 4805 */ 4806 if (period == 0) 4807 return 0; 4808 4809 return div64_u64(runtime << BW_SHIFT, period); 4810 } 4811 4812 /* 4813 * wake_up_new_task - wake up a newly created task for the first time. 4814 * 4815 * This function will do some initial scheduler statistics housekeeping 4816 * that must be done for every newly created context, then puts the task 4817 * on the runqueue and wakes it. 4818 */ 4819 void wake_up_new_task(struct task_struct *p) 4820 { 4821 struct rq_flags rf; 4822 struct rq *rq; 4823 int wake_flags = WF_FORK; 4824 4825 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4826 WRITE_ONCE(p->__state, TASK_RUNNING); 4827 /* 4828 * Fork balancing, do it here and not earlier because: 4829 * - cpus_ptr can change in the fork path 4830 * - any previously selected CPU might disappear through hotplug 4831 * 4832 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4833 * as we're not fully set-up yet. 4834 */ 4835 p->recent_used_cpu = task_cpu(p); 4836 rseq_migrate(p); 4837 __set_task_cpu(p, select_task_rq(p, task_cpu(p), &wake_flags)); 4838 rq = __task_rq_lock(p, &rf); 4839 update_rq_clock(rq); 4840 post_init_entity_util_avg(p); 4841 4842 activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL); 4843 trace_sched_wakeup_new(p); 4844 wakeup_preempt(rq, p, wake_flags); 4845 if (p->sched_class->task_woken) { 4846 /* 4847 * Nothing relies on rq->lock after this, so it's fine to 4848 * drop it. 4849 */ 4850 rq_unpin_lock(rq, &rf); 4851 p->sched_class->task_woken(rq, p); 4852 rq_repin_lock(rq, &rf); 4853 } 4854 task_rq_unlock(rq, p, &rf); 4855 } 4856 4857 #ifdef CONFIG_PREEMPT_NOTIFIERS 4858 4859 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4860 4861 void preempt_notifier_inc(void) 4862 { 4863 static_branch_inc(&preempt_notifier_key); 4864 } 4865 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4866 4867 void preempt_notifier_dec(void) 4868 { 4869 static_branch_dec(&preempt_notifier_key); 4870 } 4871 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4872 4873 /** 4874 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4875 * @notifier: notifier struct to register 4876 */ 4877 void preempt_notifier_register(struct preempt_notifier *notifier) 4878 { 4879 if (!static_branch_unlikely(&preempt_notifier_key)) 4880 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4881 4882 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4883 } 4884 EXPORT_SYMBOL_GPL(preempt_notifier_register); 4885 4886 /** 4887 * preempt_notifier_unregister - no longer interested in preemption notifications 4888 * @notifier: notifier struct to unregister 4889 * 4890 * This is *not* safe to call from within a preemption notifier. 4891 */ 4892 void preempt_notifier_unregister(struct preempt_notifier *notifier) 4893 { 4894 hlist_del(¬ifier->link); 4895 } 4896 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4897 4898 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4899 { 4900 struct preempt_notifier *notifier; 4901 4902 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4903 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4904 } 4905 4906 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4907 { 4908 if (static_branch_unlikely(&preempt_notifier_key)) 4909 __fire_sched_in_preempt_notifiers(curr); 4910 } 4911 4912 static void 4913 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 4914 struct task_struct *next) 4915 { 4916 struct preempt_notifier *notifier; 4917 4918 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4919 notifier->ops->sched_out(notifier, next); 4920 } 4921 4922 static __always_inline void 4923 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4924 struct task_struct *next) 4925 { 4926 if (static_branch_unlikely(&preempt_notifier_key)) 4927 __fire_sched_out_preempt_notifiers(curr, next); 4928 } 4929 4930 #else /* !CONFIG_PREEMPT_NOTIFIERS: */ 4931 4932 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4933 { 4934 } 4935 4936 static inline void 4937 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4938 struct task_struct *next) 4939 { 4940 } 4941 4942 #endif /* !CONFIG_PREEMPT_NOTIFIERS */ 4943 4944 static inline void prepare_task(struct task_struct *next) 4945 { 4946 /* 4947 * Claim the task as running, we do this before switching to it 4948 * such that any running task will have this set. 4949 * 4950 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and 4951 * its ordering comment. 4952 */ 4953 WRITE_ONCE(next->on_cpu, 1); 4954 } 4955 4956 static inline void finish_task(struct task_struct *prev) 4957 { 4958 /* 4959 * This must be the very last reference to @prev from this CPU. After 4960 * p->on_cpu is cleared, the task can be moved to a different CPU. We 4961 * must ensure this doesn't happen until the switch is completely 4962 * finished. 4963 * 4964 * In particular, the load of prev->state in finish_task_switch() must 4965 * happen before this. 4966 * 4967 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 4968 */ 4969 smp_store_release(&prev->on_cpu, 0); 4970 } 4971 4972 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) 4973 { 4974 void (*func)(struct rq *rq); 4975 struct balance_callback *next; 4976 4977 lockdep_assert_rq_held(rq); 4978 4979 while (head) { 4980 func = (void (*)(struct rq *))head->func; 4981 next = head->next; 4982 head->next = NULL; 4983 head = next; 4984 4985 func(rq); 4986 } 4987 } 4988 4989 static void balance_push(struct rq *rq); 4990 4991 /* 4992 * balance_push_callback is a right abuse of the callback interface and plays 4993 * by significantly different rules. 4994 * 4995 * Where the normal balance_callback's purpose is to be ran in the same context 4996 * that queued it (only later, when it's safe to drop rq->lock again), 4997 * balance_push_callback is specifically targeted at __schedule(). 4998 * 4999 * This abuse is tolerated because it places all the unlikely/odd cases behind 5000 * a single test, namely: rq->balance_callback == NULL. 5001 */ 5002 struct balance_callback balance_push_callback = { 5003 .next = NULL, 5004 .func = balance_push, 5005 }; 5006 5007 static inline struct balance_callback * 5008 __splice_balance_callbacks(struct rq *rq, bool split) 5009 { 5010 struct balance_callback *head = rq->balance_callback; 5011 5012 if (likely(!head)) 5013 return NULL; 5014 5015 lockdep_assert_rq_held(rq); 5016 /* 5017 * Must not take balance_push_callback off the list when 5018 * splice_balance_callbacks() and balance_callbacks() are not 5019 * in the same rq->lock section. 5020 * 5021 * In that case it would be possible for __schedule() to interleave 5022 * and observe the list empty. 5023 */ 5024 if (split && head == &balance_push_callback) 5025 head = NULL; 5026 else 5027 rq->balance_callback = NULL; 5028 5029 return head; 5030 } 5031 5032 struct balance_callback *splice_balance_callbacks(struct rq *rq) 5033 { 5034 return __splice_balance_callbacks(rq, true); 5035 } 5036 5037 static void __balance_callbacks(struct rq *rq) 5038 { 5039 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); 5040 } 5041 5042 void balance_callbacks(struct rq *rq, struct balance_callback *head) 5043 { 5044 unsigned long flags; 5045 5046 if (unlikely(head)) { 5047 raw_spin_rq_lock_irqsave(rq, flags); 5048 do_balance_callbacks(rq, head); 5049 raw_spin_rq_unlock_irqrestore(rq, flags); 5050 } 5051 } 5052 5053 static inline void 5054 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 5055 { 5056 /* 5057 * Since the runqueue lock will be released by the next 5058 * task (which is an invalid locking op but in the case 5059 * of the scheduler it's an obvious special-case), so we 5060 * do an early lockdep release here: 5061 */ 5062 rq_unpin_lock(rq, rf); 5063 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 5064 #ifdef CONFIG_DEBUG_SPINLOCK 5065 /* this is a valid case when another task releases the spinlock */ 5066 rq_lockp(rq)->owner = next; 5067 #endif 5068 } 5069 5070 static inline void finish_lock_switch(struct rq *rq) 5071 { 5072 /* 5073 * If we are tracking spinlock dependencies then we have to 5074 * fix up the runqueue lock - which gets 'carried over' from 5075 * prev into current: 5076 */ 5077 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 5078 __balance_callbacks(rq); 5079 raw_spin_rq_unlock_irq(rq); 5080 } 5081 5082 /* 5083 * NOP if the arch has not defined these: 5084 */ 5085 5086 #ifndef prepare_arch_switch 5087 # define prepare_arch_switch(next) do { } while (0) 5088 #endif 5089 5090 #ifndef finish_arch_post_lock_switch 5091 # define finish_arch_post_lock_switch() do { } while (0) 5092 #endif 5093 5094 static inline void kmap_local_sched_out(void) 5095 { 5096 #ifdef CONFIG_KMAP_LOCAL 5097 if (unlikely(current->kmap_ctrl.idx)) 5098 __kmap_local_sched_out(); 5099 #endif 5100 } 5101 5102 static inline void kmap_local_sched_in(void) 5103 { 5104 #ifdef CONFIG_KMAP_LOCAL 5105 if (unlikely(current->kmap_ctrl.idx)) 5106 __kmap_local_sched_in(); 5107 #endif 5108 } 5109 5110 /** 5111 * prepare_task_switch - prepare to switch tasks 5112 * @rq: the runqueue preparing to switch 5113 * @prev: the current task that is being switched out 5114 * @next: the task we are going to switch to. 5115 * 5116 * This is called with the rq lock held and interrupts off. It must 5117 * be paired with a subsequent finish_task_switch after the context 5118 * switch. 5119 * 5120 * prepare_task_switch sets up locking and calls architecture specific 5121 * hooks. 5122 */ 5123 static inline void 5124 prepare_task_switch(struct rq *rq, struct task_struct *prev, 5125 struct task_struct *next) 5126 { 5127 kcov_prepare_switch(prev); 5128 sched_info_switch(rq, prev, next); 5129 perf_event_task_sched_out(prev, next); 5130 rseq_preempt(prev); 5131 fire_sched_out_preempt_notifiers(prev, next); 5132 kmap_local_sched_out(); 5133 prepare_task(next); 5134 prepare_arch_switch(next); 5135 } 5136 5137 /** 5138 * finish_task_switch - clean up after a task-switch 5139 * @prev: the thread we just switched away from. 5140 * 5141 * finish_task_switch must be called after the context switch, paired 5142 * with a prepare_task_switch call before the context switch. 5143 * finish_task_switch will reconcile locking set up by prepare_task_switch, 5144 * and do any other architecture-specific cleanup actions. 5145 * 5146 * Note that we may have delayed dropping an mm in context_switch(). If 5147 * so, we finish that here outside of the runqueue lock. (Doing it 5148 * with the lock held can cause deadlocks; see schedule() for 5149 * details.) 5150 * 5151 * The context switch have flipped the stack from under us and restored the 5152 * local variables which were saved when this task called schedule() in the 5153 * past. 'prev == current' is still correct but we need to recalculate this_rq 5154 * because prev may have moved to another CPU. 5155 */ 5156 static struct rq *finish_task_switch(struct task_struct *prev) 5157 __releases(rq->lock) 5158 { 5159 struct rq *rq = this_rq(); 5160 struct mm_struct *mm = rq->prev_mm; 5161 unsigned int prev_state; 5162 5163 /* 5164 * The previous task will have left us with a preempt_count of 2 5165 * because it left us after: 5166 * 5167 * schedule() 5168 * preempt_disable(); // 1 5169 * __schedule() 5170 * raw_spin_lock_irq(&rq->lock) // 2 5171 * 5172 * Also, see FORK_PREEMPT_COUNT. 5173 */ 5174 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 5175 "corrupted preempt_count: %s/%d/0x%x\n", 5176 current->comm, current->pid, preempt_count())) 5177 preempt_count_set(FORK_PREEMPT_COUNT); 5178 5179 rq->prev_mm = NULL; 5180 5181 /* 5182 * A task struct has one reference for the use as "current". 5183 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 5184 * schedule one last time. The schedule call will never return, and 5185 * the scheduled task must drop that reference. 5186 * 5187 * We must observe prev->state before clearing prev->on_cpu (in 5188 * finish_task), otherwise a concurrent wakeup can get prev 5189 * running on another CPU and we could rave with its RUNNING -> DEAD 5190 * transition, resulting in a double drop. 5191 */ 5192 prev_state = READ_ONCE(prev->__state); 5193 vtime_task_switch(prev); 5194 perf_event_task_sched_in(prev, current); 5195 finish_task(prev); 5196 tick_nohz_task_switch(); 5197 finish_lock_switch(rq); 5198 finish_arch_post_lock_switch(); 5199 kcov_finish_switch(current); 5200 /* 5201 * kmap_local_sched_out() is invoked with rq::lock held and 5202 * interrupts disabled. There is no requirement for that, but the 5203 * sched out code does not have an interrupt enabled section. 5204 * Restoring the maps on sched in does not require interrupts being 5205 * disabled either. 5206 */ 5207 kmap_local_sched_in(); 5208 5209 fire_sched_in_preempt_notifiers(current); 5210 /* 5211 * When switching through a kernel thread, the loop in 5212 * membarrier_{private,global}_expedited() may have observed that 5213 * kernel thread and not issued an IPI. It is therefore possible to 5214 * schedule between user->kernel->user threads without passing though 5215 * switch_mm(). Membarrier requires a barrier after storing to 5216 * rq->curr, before returning to userspace, so provide them here: 5217 * 5218 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 5219 * provided by mmdrop_lazy_tlb(), 5220 * - a sync_core for SYNC_CORE. 5221 */ 5222 if (mm) { 5223 membarrier_mm_sync_core_before_usermode(mm); 5224 mmdrop_lazy_tlb_sched(mm); 5225 } 5226 5227 if (unlikely(prev_state == TASK_DEAD)) { 5228 if (prev->sched_class->task_dead) 5229 prev->sched_class->task_dead(prev); 5230 5231 /* Task is done with its stack. */ 5232 put_task_stack(prev); 5233 5234 put_task_struct_rcu_user(prev); 5235 } 5236 5237 return rq; 5238 } 5239 5240 /** 5241 * schedule_tail - first thing a freshly forked thread must call. 5242 * @prev: the thread we just switched away from. 5243 */ 5244 asmlinkage __visible void schedule_tail(struct task_struct *prev) 5245 __releases(rq->lock) 5246 { 5247 /* 5248 * New tasks start with FORK_PREEMPT_COUNT, see there and 5249 * finish_task_switch() for details. 5250 * 5251 * finish_task_switch() will drop rq->lock() and lower preempt_count 5252 * and the preempt_enable() will end up enabling preemption (on 5253 * PREEMPT_COUNT kernels). 5254 */ 5255 5256 finish_task_switch(prev); 5257 /* 5258 * This is a special case: the newly created task has just 5259 * switched the context for the first time. It is returning from 5260 * schedule for the first time in this path. 5261 */ 5262 trace_sched_exit_tp(true); 5263 preempt_enable(); 5264 5265 if (current->set_child_tid) 5266 put_user(task_pid_vnr(current), current->set_child_tid); 5267 5268 calculate_sigpending(); 5269 } 5270 5271 /* 5272 * context_switch - switch to the new MM and the new thread's register state. 5273 */ 5274 static __always_inline struct rq * 5275 context_switch(struct rq *rq, struct task_struct *prev, 5276 struct task_struct *next, struct rq_flags *rf) 5277 { 5278 prepare_task_switch(rq, prev, next); 5279 5280 /* 5281 * For paravirt, this is coupled with an exit in switch_to to 5282 * combine the page table reload and the switch backend into 5283 * one hypercall. 5284 */ 5285 arch_start_context_switch(prev); 5286 5287 /* 5288 * kernel -> kernel lazy + transfer active 5289 * user -> kernel lazy + mmgrab_lazy_tlb() active 5290 * 5291 * kernel -> user switch + mmdrop_lazy_tlb() active 5292 * user -> user switch 5293 * 5294 * switch_mm_cid() needs to be updated if the barriers provided 5295 * by context_switch() are modified. 5296 */ 5297 if (!next->mm) { // to kernel 5298 enter_lazy_tlb(prev->active_mm, next); 5299 5300 next->active_mm = prev->active_mm; 5301 if (prev->mm) // from user 5302 mmgrab_lazy_tlb(prev->active_mm); 5303 else 5304 prev->active_mm = NULL; 5305 } else { // to user 5306 membarrier_switch_mm(rq, prev->active_mm, next->mm); 5307 /* 5308 * sys_membarrier() requires an smp_mb() between setting 5309 * rq->curr / membarrier_switch_mm() and returning to userspace. 5310 * 5311 * The below provides this either through switch_mm(), or in 5312 * case 'prev->active_mm == next->mm' through 5313 * finish_task_switch()'s mmdrop(). 5314 */ 5315 switch_mm_irqs_off(prev->active_mm, next->mm, next); 5316 lru_gen_use_mm(next->mm); 5317 5318 if (!prev->mm) { // from kernel 5319 /* will mmdrop_lazy_tlb() in finish_task_switch(). */ 5320 rq->prev_mm = prev->active_mm; 5321 prev->active_mm = NULL; 5322 } 5323 } 5324 5325 /* switch_mm_cid() requires the memory barriers above. */ 5326 switch_mm_cid(rq, prev, next); 5327 5328 prepare_lock_switch(rq, next, rf); 5329 5330 /* Here we just switch the register state and the stack. */ 5331 switch_to(prev, next, prev); 5332 barrier(); 5333 5334 return finish_task_switch(prev); 5335 } 5336 5337 /* 5338 * nr_running and nr_context_switches: 5339 * 5340 * externally visible scheduler statistics: current number of runnable 5341 * threads, total number of context switches performed since bootup. 5342 */ 5343 unsigned int nr_running(void) 5344 { 5345 unsigned int i, sum = 0; 5346 5347 for_each_online_cpu(i) 5348 sum += cpu_rq(i)->nr_running; 5349 5350 return sum; 5351 } 5352 5353 /* 5354 * Check if only the current task is running on the CPU. 5355 * 5356 * Caution: this function does not check that the caller has disabled 5357 * preemption, thus the result might have a time-of-check-to-time-of-use 5358 * race. The caller is responsible to use it correctly, for example: 5359 * 5360 * - from a non-preemptible section (of course) 5361 * 5362 * - from a thread that is bound to a single CPU 5363 * 5364 * - in a loop with very short iterations (e.g. a polling loop) 5365 */ 5366 bool single_task_running(void) 5367 { 5368 return raw_rq()->nr_running == 1; 5369 } 5370 EXPORT_SYMBOL(single_task_running); 5371 5372 unsigned long long nr_context_switches_cpu(int cpu) 5373 { 5374 return cpu_rq(cpu)->nr_switches; 5375 } 5376 5377 unsigned long long nr_context_switches(void) 5378 { 5379 int i; 5380 unsigned long long sum = 0; 5381 5382 for_each_possible_cpu(i) 5383 sum += cpu_rq(i)->nr_switches; 5384 5385 return sum; 5386 } 5387 5388 /* 5389 * Consumers of these two interfaces, like for example the cpuidle menu 5390 * governor, are using nonsensical data. Preferring shallow idle state selection 5391 * for a CPU that has IO-wait which might not even end up running the task when 5392 * it does become runnable. 5393 */ 5394 5395 unsigned int nr_iowait_cpu(int cpu) 5396 { 5397 return atomic_read(&cpu_rq(cpu)->nr_iowait); 5398 } 5399 5400 /* 5401 * IO-wait accounting, and how it's mostly bollocks (on SMP). 5402 * 5403 * The idea behind IO-wait account is to account the idle time that we could 5404 * have spend running if it were not for IO. That is, if we were to improve the 5405 * storage performance, we'd have a proportional reduction in IO-wait time. 5406 * 5407 * This all works nicely on UP, where, when a task blocks on IO, we account 5408 * idle time as IO-wait, because if the storage were faster, it could've been 5409 * running and we'd not be idle. 5410 * 5411 * This has been extended to SMP, by doing the same for each CPU. This however 5412 * is broken. 5413 * 5414 * Imagine for instance the case where two tasks block on one CPU, only the one 5415 * CPU will have IO-wait accounted, while the other has regular idle. Even 5416 * though, if the storage were faster, both could've ran at the same time, 5417 * utilising both CPUs. 5418 * 5419 * This means, that when looking globally, the current IO-wait accounting on 5420 * SMP is a lower bound, by reason of under accounting. 5421 * 5422 * Worse, since the numbers are provided per CPU, they are sometimes 5423 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 5424 * associated with any one particular CPU, it can wake to another CPU than it 5425 * blocked on. This means the per CPU IO-wait number is meaningless. 5426 * 5427 * Task CPU affinities can make all that even more 'interesting'. 5428 */ 5429 5430 unsigned int nr_iowait(void) 5431 { 5432 unsigned int i, sum = 0; 5433 5434 for_each_possible_cpu(i) 5435 sum += nr_iowait_cpu(i); 5436 5437 return sum; 5438 } 5439 5440 /* 5441 * sched_exec - execve() is a valuable balancing opportunity, because at 5442 * this point the task has the smallest effective memory and cache footprint. 5443 */ 5444 void sched_exec(void) 5445 { 5446 struct task_struct *p = current; 5447 struct migration_arg arg; 5448 int dest_cpu; 5449 5450 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 5451 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 5452 if (dest_cpu == smp_processor_id()) 5453 return; 5454 5455 if (unlikely(!cpu_active(dest_cpu))) 5456 return; 5457 5458 arg = (struct migration_arg){ p, dest_cpu }; 5459 } 5460 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 5461 } 5462 5463 DEFINE_PER_CPU(struct kernel_stat, kstat); 5464 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 5465 5466 EXPORT_PER_CPU_SYMBOL(kstat); 5467 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 5468 5469 /* 5470 * The function fair_sched_class.update_curr accesses the struct curr 5471 * and its field curr->exec_start; when called from task_sched_runtime(), 5472 * we observe a high rate of cache misses in practice. 5473 * Prefetching this data results in improved performance. 5474 */ 5475 static inline void prefetch_curr_exec_start(struct task_struct *p) 5476 { 5477 #ifdef CONFIG_FAIR_GROUP_SCHED 5478 struct sched_entity *curr = p->se.cfs_rq->curr; 5479 #else 5480 struct sched_entity *curr = task_rq(p)->cfs.curr; 5481 #endif 5482 prefetch(curr); 5483 prefetch(&curr->exec_start); 5484 } 5485 5486 /* 5487 * Return accounted runtime for the task. 5488 * In case the task is currently running, return the runtime plus current's 5489 * pending runtime that have not been accounted yet. 5490 */ 5491 unsigned long long task_sched_runtime(struct task_struct *p) 5492 { 5493 struct rq_flags rf; 5494 struct rq *rq; 5495 u64 ns; 5496 5497 #ifdef CONFIG_64BIT 5498 /* 5499 * 64-bit doesn't need locks to atomically read a 64-bit value. 5500 * So we have a optimization chance when the task's delta_exec is 0. 5501 * Reading ->on_cpu is racy, but this is OK. 5502 * 5503 * If we race with it leaving CPU, we'll take a lock. So we're correct. 5504 * If we race with it entering CPU, unaccounted time is 0. This is 5505 * indistinguishable from the read occurring a few cycles earlier. 5506 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 5507 * been accounted, so we're correct here as well. 5508 */ 5509 if (!p->on_cpu || !task_on_rq_queued(p)) 5510 return p->se.sum_exec_runtime; 5511 #endif 5512 5513 rq = task_rq_lock(p, &rf); 5514 /* 5515 * Must be ->curr _and_ ->on_rq. If dequeued, we would 5516 * project cycles that may never be accounted to this 5517 * thread, breaking clock_gettime(). 5518 */ 5519 if (task_current_donor(rq, p) && task_on_rq_queued(p)) { 5520 prefetch_curr_exec_start(p); 5521 update_rq_clock(rq); 5522 p->sched_class->update_curr(rq); 5523 } 5524 ns = p->se.sum_exec_runtime; 5525 task_rq_unlock(rq, p, &rf); 5526 5527 return ns; 5528 } 5529 5530 static u64 cpu_resched_latency(struct rq *rq) 5531 { 5532 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 5533 u64 resched_latency, now = rq_clock(rq); 5534 static bool warned_once; 5535 5536 if (sysctl_resched_latency_warn_once && warned_once) 5537 return 0; 5538 5539 if (!need_resched() || !latency_warn_ms) 5540 return 0; 5541 5542 if (system_state == SYSTEM_BOOTING) 5543 return 0; 5544 5545 if (!rq->last_seen_need_resched_ns) { 5546 rq->last_seen_need_resched_ns = now; 5547 rq->ticks_without_resched = 0; 5548 return 0; 5549 } 5550 5551 rq->ticks_without_resched++; 5552 resched_latency = now - rq->last_seen_need_resched_ns; 5553 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 5554 return 0; 5555 5556 warned_once = true; 5557 5558 return resched_latency; 5559 } 5560 5561 static int __init setup_resched_latency_warn_ms(char *str) 5562 { 5563 long val; 5564 5565 if ((kstrtol(str, 0, &val))) { 5566 pr_warn("Unable to set resched_latency_warn_ms\n"); 5567 return 1; 5568 } 5569 5570 sysctl_resched_latency_warn_ms = val; 5571 return 1; 5572 } 5573 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 5574 5575 /* 5576 * This function gets called by the timer code, with HZ frequency. 5577 * We call it with interrupts disabled. 5578 */ 5579 void sched_tick(void) 5580 { 5581 int cpu = smp_processor_id(); 5582 struct rq *rq = cpu_rq(cpu); 5583 /* accounting goes to the donor task */ 5584 struct task_struct *donor; 5585 struct rq_flags rf; 5586 unsigned long hw_pressure; 5587 u64 resched_latency; 5588 5589 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5590 arch_scale_freq_tick(); 5591 5592 sched_clock_tick(); 5593 5594 rq_lock(rq, &rf); 5595 donor = rq->donor; 5596 5597 psi_account_irqtime(rq, donor, NULL); 5598 5599 update_rq_clock(rq); 5600 hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); 5601 update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure); 5602 5603 if (dynamic_preempt_lazy() && tif_test_bit(TIF_NEED_RESCHED_LAZY)) 5604 resched_curr(rq); 5605 5606 donor->sched_class->task_tick(rq, donor, 0); 5607 if (sched_feat(LATENCY_WARN)) 5608 resched_latency = cpu_resched_latency(rq); 5609 calc_global_load_tick(rq); 5610 sched_core_tick(rq); 5611 task_tick_mm_cid(rq, donor); 5612 scx_tick(rq); 5613 5614 rq_unlock(rq, &rf); 5615 5616 if (sched_feat(LATENCY_WARN) && resched_latency) 5617 resched_latency_warn(cpu, resched_latency); 5618 5619 perf_event_task_tick(); 5620 5621 if (donor->flags & PF_WQ_WORKER) 5622 wq_worker_tick(donor); 5623 5624 if (!scx_switched_all()) { 5625 rq->idle_balance = idle_cpu(cpu); 5626 sched_balance_trigger(rq); 5627 } 5628 } 5629 5630 #ifdef CONFIG_NO_HZ_FULL 5631 5632 struct tick_work { 5633 int cpu; 5634 atomic_t state; 5635 struct delayed_work work; 5636 }; 5637 /* Values for ->state, see diagram below. */ 5638 #define TICK_SCHED_REMOTE_OFFLINE 0 5639 #define TICK_SCHED_REMOTE_OFFLINING 1 5640 #define TICK_SCHED_REMOTE_RUNNING 2 5641 5642 /* 5643 * State diagram for ->state: 5644 * 5645 * 5646 * TICK_SCHED_REMOTE_OFFLINE 5647 * | ^ 5648 * | | 5649 * | | sched_tick_remote() 5650 * | | 5651 * | | 5652 * +--TICK_SCHED_REMOTE_OFFLINING 5653 * | ^ 5654 * | | 5655 * sched_tick_start() | | sched_tick_stop() 5656 * | | 5657 * V | 5658 * TICK_SCHED_REMOTE_RUNNING 5659 * 5660 * 5661 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5662 * and sched_tick_start() are happy to leave the state in RUNNING. 5663 */ 5664 5665 static struct tick_work __percpu *tick_work_cpu; 5666 5667 static void sched_tick_remote(struct work_struct *work) 5668 { 5669 struct delayed_work *dwork = to_delayed_work(work); 5670 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5671 int cpu = twork->cpu; 5672 struct rq *rq = cpu_rq(cpu); 5673 int os; 5674 5675 /* 5676 * Handle the tick only if it appears the remote CPU is running in full 5677 * dynticks mode. The check is racy by nature, but missing a tick or 5678 * having one too much is no big deal because the scheduler tick updates 5679 * statistics and checks timeslices in a time-independent way, regardless 5680 * of when exactly it is running. 5681 */ 5682 if (tick_nohz_tick_stopped_cpu(cpu)) { 5683 guard(rq_lock_irq)(rq); 5684 struct task_struct *curr = rq->curr; 5685 5686 if (cpu_online(cpu)) { 5687 /* 5688 * Since this is a remote tick for full dynticks mode, 5689 * we are always sure that there is no proxy (only a 5690 * single task is running). 5691 */ 5692 WARN_ON_ONCE(rq->curr != rq->donor); 5693 update_rq_clock(rq); 5694 5695 if (!is_idle_task(curr)) { 5696 /* 5697 * Make sure the next tick runs within a 5698 * reasonable amount of time. 5699 */ 5700 u64 delta = rq_clock_task(rq) - curr->se.exec_start; 5701 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 5702 } 5703 curr->sched_class->task_tick(rq, curr, 0); 5704 5705 calc_load_nohz_remote(rq); 5706 } 5707 } 5708 5709 /* 5710 * Run the remote tick once per second (1Hz). This arbitrary 5711 * frequency is large enough to avoid overload but short enough 5712 * to keep scheduler internal stats reasonably up to date. But 5713 * first update state to reflect hotplug activity if required. 5714 */ 5715 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5716 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5717 if (os == TICK_SCHED_REMOTE_RUNNING) 5718 queue_delayed_work(system_unbound_wq, dwork, HZ); 5719 } 5720 5721 static void sched_tick_start(int cpu) 5722 { 5723 int os; 5724 struct tick_work *twork; 5725 5726 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5727 return; 5728 5729 WARN_ON_ONCE(!tick_work_cpu); 5730 5731 twork = per_cpu_ptr(tick_work_cpu, cpu); 5732 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5733 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5734 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5735 twork->cpu = cpu; 5736 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5737 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5738 } 5739 } 5740 5741 #ifdef CONFIG_HOTPLUG_CPU 5742 static void sched_tick_stop(int cpu) 5743 { 5744 struct tick_work *twork; 5745 int os; 5746 5747 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5748 return; 5749 5750 WARN_ON_ONCE(!tick_work_cpu); 5751 5752 twork = per_cpu_ptr(tick_work_cpu, cpu); 5753 /* There cannot be competing actions, but don't rely on stop-machine. */ 5754 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5755 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5756 /* Don't cancel, as this would mess up the state machine. */ 5757 } 5758 #endif /* CONFIG_HOTPLUG_CPU */ 5759 5760 int __init sched_tick_offload_init(void) 5761 { 5762 tick_work_cpu = alloc_percpu(struct tick_work); 5763 BUG_ON(!tick_work_cpu); 5764 return 0; 5765 } 5766 5767 #else /* !CONFIG_NO_HZ_FULL: */ 5768 static inline void sched_tick_start(int cpu) { } 5769 static inline void sched_tick_stop(int cpu) { } 5770 #endif /* !CONFIG_NO_HZ_FULL */ 5771 5772 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5773 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5774 /* 5775 * If the value passed in is equal to the current preempt count 5776 * then we just disabled preemption. Start timing the latency. 5777 */ 5778 static inline void preempt_latency_start(int val) 5779 { 5780 if (preempt_count() == val) { 5781 unsigned long ip = get_lock_parent_ip(); 5782 #ifdef CONFIG_DEBUG_PREEMPT 5783 current->preempt_disable_ip = ip; 5784 #endif 5785 trace_preempt_off(CALLER_ADDR0, ip); 5786 } 5787 } 5788 5789 void preempt_count_add(int val) 5790 { 5791 #ifdef CONFIG_DEBUG_PREEMPT 5792 /* 5793 * Underflow? 5794 */ 5795 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5796 return; 5797 #endif 5798 __preempt_count_add(val); 5799 #ifdef CONFIG_DEBUG_PREEMPT 5800 /* 5801 * Spinlock count overflowing soon? 5802 */ 5803 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5804 PREEMPT_MASK - 10); 5805 #endif 5806 preempt_latency_start(val); 5807 } 5808 EXPORT_SYMBOL(preempt_count_add); 5809 NOKPROBE_SYMBOL(preempt_count_add); 5810 5811 /* 5812 * If the value passed in equals to the current preempt count 5813 * then we just enabled preemption. Stop timing the latency. 5814 */ 5815 static inline void preempt_latency_stop(int val) 5816 { 5817 if (preempt_count() == val) 5818 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5819 } 5820 5821 void preempt_count_sub(int val) 5822 { 5823 #ifdef CONFIG_DEBUG_PREEMPT 5824 /* 5825 * Underflow? 5826 */ 5827 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5828 return; 5829 /* 5830 * Is the spinlock portion underflowing? 5831 */ 5832 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5833 !(preempt_count() & PREEMPT_MASK))) 5834 return; 5835 #endif 5836 5837 preempt_latency_stop(val); 5838 __preempt_count_sub(val); 5839 } 5840 EXPORT_SYMBOL(preempt_count_sub); 5841 NOKPROBE_SYMBOL(preempt_count_sub); 5842 5843 #else 5844 static inline void preempt_latency_start(int val) { } 5845 static inline void preempt_latency_stop(int val) { } 5846 #endif 5847 5848 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5849 { 5850 #ifdef CONFIG_DEBUG_PREEMPT 5851 return p->preempt_disable_ip; 5852 #else 5853 return 0; 5854 #endif 5855 } 5856 5857 /* 5858 * Print scheduling while atomic bug: 5859 */ 5860 static noinline void __schedule_bug(struct task_struct *prev) 5861 { 5862 /* Save this before calling printk(), since that will clobber it */ 5863 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5864 5865 if (oops_in_progress) 5866 return; 5867 5868 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5869 prev->comm, prev->pid, preempt_count()); 5870 5871 debug_show_held_locks(prev); 5872 print_modules(); 5873 if (irqs_disabled()) 5874 print_irqtrace_events(prev); 5875 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 5876 pr_err("Preemption disabled at:"); 5877 print_ip_sym(KERN_ERR, preempt_disable_ip); 5878 } 5879 check_panic_on_warn("scheduling while atomic"); 5880 5881 dump_stack(); 5882 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5883 } 5884 5885 /* 5886 * Various schedule()-time debugging checks and statistics: 5887 */ 5888 static inline void schedule_debug(struct task_struct *prev, bool preempt) 5889 { 5890 #ifdef CONFIG_SCHED_STACK_END_CHECK 5891 if (task_stack_end_corrupted(prev)) 5892 panic("corrupted stack end detected inside scheduler\n"); 5893 5894 if (task_scs_end_corrupted(prev)) 5895 panic("corrupted shadow stack detected inside scheduler\n"); 5896 #endif 5897 5898 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5899 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5900 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5901 prev->comm, prev->pid, prev->non_block_count); 5902 dump_stack(); 5903 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5904 } 5905 #endif 5906 5907 if (unlikely(in_atomic_preempt_off())) { 5908 __schedule_bug(prev); 5909 preempt_count_set(PREEMPT_DISABLED); 5910 } 5911 rcu_sleep_check(); 5912 WARN_ON_ONCE(ct_state() == CT_STATE_USER); 5913 5914 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5915 5916 schedstat_inc(this_rq()->sched_count); 5917 } 5918 5919 static void prev_balance(struct rq *rq, struct task_struct *prev, 5920 struct rq_flags *rf) 5921 { 5922 const struct sched_class *start_class = prev->sched_class; 5923 const struct sched_class *class; 5924 5925 #ifdef CONFIG_SCHED_CLASS_EXT 5926 /* 5927 * SCX requires a balance() call before every pick_task() including when 5928 * waking up from SCHED_IDLE. If @start_class is below SCX, start from 5929 * SCX instead. Also, set a flag to detect missing balance() call. 5930 */ 5931 if (scx_enabled()) { 5932 rq->scx.flags |= SCX_RQ_BAL_PENDING; 5933 if (sched_class_above(&ext_sched_class, start_class)) 5934 start_class = &ext_sched_class; 5935 } 5936 #endif 5937 5938 /* 5939 * We must do the balancing pass before put_prev_task(), such 5940 * that when we release the rq->lock the task is in the same 5941 * state as before we took rq->lock. 5942 * 5943 * We can terminate the balance pass as soon as we know there is 5944 * a runnable task of @class priority or higher. 5945 */ 5946 for_active_class_range(class, start_class, &idle_sched_class) { 5947 if (class->balance && class->balance(rq, prev, rf)) 5948 break; 5949 } 5950 } 5951 5952 /* 5953 * Pick up the highest-prio task: 5954 */ 5955 static inline struct task_struct * 5956 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5957 { 5958 const struct sched_class *class; 5959 struct task_struct *p; 5960 5961 rq->dl_server = NULL; 5962 5963 if (scx_enabled()) 5964 goto restart; 5965 5966 /* 5967 * Optimization: we know that if all tasks are in the fair class we can 5968 * call that function directly, but only if the @prev task wasn't of a 5969 * higher scheduling class, because otherwise those lose the 5970 * opportunity to pull in more work from other CPUs. 5971 */ 5972 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && 5973 rq->nr_running == rq->cfs.h_nr_queued)) { 5974 5975 p = pick_next_task_fair(rq, prev, rf); 5976 if (unlikely(p == RETRY_TASK)) 5977 goto restart; 5978 5979 /* Assume the next prioritized class is idle_sched_class */ 5980 if (!p) { 5981 p = pick_task_idle(rq); 5982 put_prev_set_next_task(rq, prev, p); 5983 } 5984 5985 return p; 5986 } 5987 5988 restart: 5989 prev_balance(rq, prev, rf); 5990 5991 for_each_active_class(class) { 5992 if (class->pick_next_task) { 5993 p = class->pick_next_task(rq, prev); 5994 if (p) 5995 return p; 5996 } else { 5997 p = class->pick_task(rq); 5998 if (p) { 5999 put_prev_set_next_task(rq, prev, p); 6000 return p; 6001 } 6002 } 6003 } 6004 6005 BUG(); /* The idle class should always have a runnable task. */ 6006 } 6007 6008 #ifdef CONFIG_SCHED_CORE 6009 static inline bool is_task_rq_idle(struct task_struct *t) 6010 { 6011 return (task_rq(t)->idle == t); 6012 } 6013 6014 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 6015 { 6016 return is_task_rq_idle(a) || (a->core_cookie == cookie); 6017 } 6018 6019 static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 6020 { 6021 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 6022 return true; 6023 6024 return a->core_cookie == b->core_cookie; 6025 } 6026 6027 static inline struct task_struct *pick_task(struct rq *rq) 6028 { 6029 const struct sched_class *class; 6030 struct task_struct *p; 6031 6032 rq->dl_server = NULL; 6033 6034 for_each_active_class(class) { 6035 p = class->pick_task(rq); 6036 if (p) 6037 return p; 6038 } 6039 6040 BUG(); /* The idle class should always have a runnable task. */ 6041 } 6042 6043 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 6044 6045 static void queue_core_balance(struct rq *rq); 6046 6047 static struct task_struct * 6048 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6049 { 6050 struct task_struct *next, *p, *max = NULL; 6051 const struct cpumask *smt_mask; 6052 bool fi_before = false; 6053 bool core_clock_updated = (rq == rq->core); 6054 unsigned long cookie; 6055 int i, cpu, occ = 0; 6056 struct rq *rq_i; 6057 bool need_sync; 6058 6059 if (!sched_core_enabled(rq)) 6060 return __pick_next_task(rq, prev, rf); 6061 6062 cpu = cpu_of(rq); 6063 6064 /* Stopper task is switching into idle, no need core-wide selection. */ 6065 if (cpu_is_offline(cpu)) { 6066 /* 6067 * Reset core_pick so that we don't enter the fastpath when 6068 * coming online. core_pick would already be migrated to 6069 * another cpu during offline. 6070 */ 6071 rq->core_pick = NULL; 6072 rq->core_dl_server = NULL; 6073 return __pick_next_task(rq, prev, rf); 6074 } 6075 6076 /* 6077 * If there were no {en,de}queues since we picked (IOW, the task 6078 * pointers are all still valid), and we haven't scheduled the last 6079 * pick yet, do so now. 6080 * 6081 * rq->core_pick can be NULL if no selection was made for a CPU because 6082 * it was either offline or went offline during a sibling's core-wide 6083 * selection. In this case, do a core-wide selection. 6084 */ 6085 if (rq->core->core_pick_seq == rq->core->core_task_seq && 6086 rq->core->core_pick_seq != rq->core_sched_seq && 6087 rq->core_pick) { 6088 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6089 6090 next = rq->core_pick; 6091 rq->dl_server = rq->core_dl_server; 6092 rq->core_pick = NULL; 6093 rq->core_dl_server = NULL; 6094 goto out_set_next; 6095 } 6096 6097 prev_balance(rq, prev, rf); 6098 6099 smt_mask = cpu_smt_mask(cpu); 6100 need_sync = !!rq->core->core_cookie; 6101 6102 /* reset state */ 6103 rq->core->core_cookie = 0UL; 6104 if (rq->core->core_forceidle_count) { 6105 if (!core_clock_updated) { 6106 update_rq_clock(rq->core); 6107 core_clock_updated = true; 6108 } 6109 sched_core_account_forceidle(rq); 6110 /* reset after accounting force idle */ 6111 rq->core->core_forceidle_start = 0; 6112 rq->core->core_forceidle_count = 0; 6113 rq->core->core_forceidle_occupation = 0; 6114 need_sync = true; 6115 fi_before = true; 6116 } 6117 6118 /* 6119 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 6120 * 6121 * @task_seq guards the task state ({en,de}queues) 6122 * @pick_seq is the @task_seq we did a selection on 6123 * @sched_seq is the @pick_seq we scheduled 6124 * 6125 * However, preemptions can cause multiple picks on the same task set. 6126 * 'Fix' this by also increasing @task_seq for every pick. 6127 */ 6128 rq->core->core_task_seq++; 6129 6130 /* 6131 * Optimize for common case where this CPU has no cookies 6132 * and there are no cookied tasks running on siblings. 6133 */ 6134 if (!need_sync) { 6135 next = pick_task(rq); 6136 if (!next->core_cookie) { 6137 rq->core_pick = NULL; 6138 rq->core_dl_server = NULL; 6139 /* 6140 * For robustness, update the min_vruntime_fi for 6141 * unconstrained picks as well. 6142 */ 6143 WARN_ON_ONCE(fi_before); 6144 task_vruntime_update(rq, next, false); 6145 goto out_set_next; 6146 } 6147 } 6148 6149 /* 6150 * For each thread: do the regular task pick and find the max prio task 6151 * amongst them. 6152 * 6153 * Tie-break prio towards the current CPU 6154 */ 6155 for_each_cpu_wrap(i, smt_mask, cpu) { 6156 rq_i = cpu_rq(i); 6157 6158 /* 6159 * Current cpu always has its clock updated on entrance to 6160 * pick_next_task(). If the current cpu is not the core, 6161 * the core may also have been updated above. 6162 */ 6163 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6164 update_rq_clock(rq_i); 6165 6166 rq_i->core_pick = p = pick_task(rq_i); 6167 rq_i->core_dl_server = rq_i->dl_server; 6168 6169 if (!max || prio_less(max, p, fi_before)) 6170 max = p; 6171 } 6172 6173 cookie = rq->core->core_cookie = max->core_cookie; 6174 6175 /* 6176 * For each thread: try and find a runnable task that matches @max or 6177 * force idle. 6178 */ 6179 for_each_cpu(i, smt_mask) { 6180 rq_i = cpu_rq(i); 6181 p = rq_i->core_pick; 6182 6183 if (!cookie_equals(p, cookie)) { 6184 p = NULL; 6185 if (cookie) 6186 p = sched_core_find(rq_i, cookie); 6187 if (!p) 6188 p = idle_sched_class.pick_task(rq_i); 6189 } 6190 6191 rq_i->core_pick = p; 6192 rq_i->core_dl_server = NULL; 6193 6194 if (p == rq_i->idle) { 6195 if (rq_i->nr_running) { 6196 rq->core->core_forceidle_count++; 6197 if (!fi_before) 6198 rq->core->core_forceidle_seq++; 6199 } 6200 } else { 6201 occ++; 6202 } 6203 } 6204 6205 if (schedstat_enabled() && rq->core->core_forceidle_count) { 6206 rq->core->core_forceidle_start = rq_clock(rq->core); 6207 rq->core->core_forceidle_occupation = occ; 6208 } 6209 6210 rq->core->core_pick_seq = rq->core->core_task_seq; 6211 next = rq->core_pick; 6212 rq->core_sched_seq = rq->core->core_pick_seq; 6213 6214 /* Something should have been selected for current CPU */ 6215 WARN_ON_ONCE(!next); 6216 6217 /* 6218 * Reschedule siblings 6219 * 6220 * NOTE: L1TF -- at this point we're no longer running the old task and 6221 * sending an IPI (below) ensures the sibling will no longer be running 6222 * their task. This ensures there is no inter-sibling overlap between 6223 * non-matching user state. 6224 */ 6225 for_each_cpu(i, smt_mask) { 6226 rq_i = cpu_rq(i); 6227 6228 /* 6229 * An online sibling might have gone offline before a task 6230 * could be picked for it, or it might be offline but later 6231 * happen to come online, but its too late and nothing was 6232 * picked for it. That's Ok - it will pick tasks for itself, 6233 * so ignore it. 6234 */ 6235 if (!rq_i->core_pick) 6236 continue; 6237 6238 /* 6239 * Update for new !FI->FI transitions, or if continuing to be in !FI: 6240 * fi_before fi update? 6241 * 0 0 1 6242 * 0 1 1 6243 * 1 0 1 6244 * 1 1 0 6245 */ 6246 if (!(fi_before && rq->core->core_forceidle_count)) 6247 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); 6248 6249 rq_i->core_pick->core_occupation = occ; 6250 6251 if (i == cpu) { 6252 rq_i->core_pick = NULL; 6253 rq_i->core_dl_server = NULL; 6254 continue; 6255 } 6256 6257 /* Did we break L1TF mitigation requirements? */ 6258 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 6259 6260 if (rq_i->curr == rq_i->core_pick) { 6261 rq_i->core_pick = NULL; 6262 rq_i->core_dl_server = NULL; 6263 continue; 6264 } 6265 6266 resched_curr(rq_i); 6267 } 6268 6269 out_set_next: 6270 put_prev_set_next_task(rq, prev, next); 6271 if (rq->core->core_forceidle_count && next == rq->idle) 6272 queue_core_balance(rq); 6273 6274 return next; 6275 } 6276 6277 static bool try_steal_cookie(int this, int that) 6278 { 6279 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 6280 struct task_struct *p; 6281 unsigned long cookie; 6282 bool success = false; 6283 6284 guard(irq)(); 6285 guard(double_rq_lock)(dst, src); 6286 6287 cookie = dst->core->core_cookie; 6288 if (!cookie) 6289 return false; 6290 6291 if (dst->curr != dst->idle) 6292 return false; 6293 6294 p = sched_core_find(src, cookie); 6295 if (!p) 6296 return false; 6297 6298 do { 6299 if (p == src->core_pick || p == src->curr) 6300 goto next; 6301 6302 if (!is_cpu_allowed(p, this)) 6303 goto next; 6304 6305 if (p->core_occupation > dst->idle->core_occupation) 6306 goto next; 6307 /* 6308 * sched_core_find() and sched_core_next() will ensure 6309 * that task @p is not throttled now, we also need to 6310 * check whether the runqueue of the destination CPU is 6311 * being throttled. 6312 */ 6313 if (sched_task_is_throttled(p, this)) 6314 goto next; 6315 6316 move_queued_task_locked(src, dst, p); 6317 resched_curr(dst); 6318 6319 success = true; 6320 break; 6321 6322 next: 6323 p = sched_core_next(p, cookie); 6324 } while (p); 6325 6326 return success; 6327 } 6328 6329 static bool steal_cookie_task(int cpu, struct sched_domain *sd) 6330 { 6331 int i; 6332 6333 for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) { 6334 if (i == cpu) 6335 continue; 6336 6337 if (need_resched()) 6338 break; 6339 6340 if (try_steal_cookie(cpu, i)) 6341 return true; 6342 } 6343 6344 return false; 6345 } 6346 6347 static void sched_core_balance(struct rq *rq) 6348 { 6349 struct sched_domain *sd; 6350 int cpu = cpu_of(rq); 6351 6352 guard(preempt)(); 6353 guard(rcu)(); 6354 6355 raw_spin_rq_unlock_irq(rq); 6356 for_each_domain(cpu, sd) { 6357 if (need_resched()) 6358 break; 6359 6360 if (steal_cookie_task(cpu, sd)) 6361 break; 6362 } 6363 raw_spin_rq_lock_irq(rq); 6364 } 6365 6366 static DEFINE_PER_CPU(struct balance_callback, core_balance_head); 6367 6368 static void queue_core_balance(struct rq *rq) 6369 { 6370 if (!sched_core_enabled(rq)) 6371 return; 6372 6373 if (!rq->core->core_cookie) 6374 return; 6375 6376 if (!rq->nr_running) /* not forced idle */ 6377 return; 6378 6379 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 6380 } 6381 6382 DEFINE_LOCK_GUARD_1(core_lock, int, 6383 sched_core_lock(*_T->lock, &_T->flags), 6384 sched_core_unlock(*_T->lock, &_T->flags), 6385 unsigned long flags) 6386 6387 static void sched_core_cpu_starting(unsigned int cpu) 6388 { 6389 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6390 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6391 int t; 6392 6393 guard(core_lock)(&cpu); 6394 6395 WARN_ON_ONCE(rq->core != rq); 6396 6397 /* if we're the first, we'll be our own leader */ 6398 if (cpumask_weight(smt_mask) == 1) 6399 return; 6400 6401 /* find the leader */ 6402 for_each_cpu(t, smt_mask) { 6403 if (t == cpu) 6404 continue; 6405 rq = cpu_rq(t); 6406 if (rq->core == rq) { 6407 core_rq = rq; 6408 break; 6409 } 6410 } 6411 6412 if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ 6413 return; 6414 6415 /* install and validate core_rq */ 6416 for_each_cpu(t, smt_mask) { 6417 rq = cpu_rq(t); 6418 6419 if (t == cpu) 6420 rq->core = core_rq; 6421 6422 WARN_ON_ONCE(rq->core != core_rq); 6423 } 6424 } 6425 6426 static void sched_core_cpu_deactivate(unsigned int cpu) 6427 { 6428 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6429 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6430 int t; 6431 6432 guard(core_lock)(&cpu); 6433 6434 /* if we're the last man standing, nothing to do */ 6435 if (cpumask_weight(smt_mask) == 1) { 6436 WARN_ON_ONCE(rq->core != rq); 6437 return; 6438 } 6439 6440 /* if we're not the leader, nothing to do */ 6441 if (rq->core != rq) 6442 return; 6443 6444 /* find a new leader */ 6445 for_each_cpu(t, smt_mask) { 6446 if (t == cpu) 6447 continue; 6448 core_rq = cpu_rq(t); 6449 break; 6450 } 6451 6452 if (WARN_ON_ONCE(!core_rq)) /* impossible */ 6453 return; 6454 6455 /* copy the shared state to the new leader */ 6456 core_rq->core_task_seq = rq->core_task_seq; 6457 core_rq->core_pick_seq = rq->core_pick_seq; 6458 core_rq->core_cookie = rq->core_cookie; 6459 core_rq->core_forceidle_count = rq->core_forceidle_count; 6460 core_rq->core_forceidle_seq = rq->core_forceidle_seq; 6461 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; 6462 6463 /* 6464 * Accounting edge for forced idle is handled in pick_next_task(). 6465 * Don't need another one here, since the hotplug thread shouldn't 6466 * have a cookie. 6467 */ 6468 core_rq->core_forceidle_start = 0; 6469 6470 /* install new leader */ 6471 for_each_cpu(t, smt_mask) { 6472 rq = cpu_rq(t); 6473 rq->core = core_rq; 6474 } 6475 } 6476 6477 static inline void sched_core_cpu_dying(unsigned int cpu) 6478 { 6479 struct rq *rq = cpu_rq(cpu); 6480 6481 if (rq->core != rq) 6482 rq->core = rq; 6483 } 6484 6485 #else /* !CONFIG_SCHED_CORE: */ 6486 6487 static inline void sched_core_cpu_starting(unsigned int cpu) {} 6488 static inline void sched_core_cpu_deactivate(unsigned int cpu) {} 6489 static inline void sched_core_cpu_dying(unsigned int cpu) {} 6490 6491 static struct task_struct * 6492 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6493 { 6494 return __pick_next_task(rq, prev, rf); 6495 } 6496 6497 #endif /* !CONFIG_SCHED_CORE */ 6498 6499 /* 6500 * Constants for the sched_mode argument of __schedule(). 6501 * 6502 * The mode argument allows RT enabled kernels to differentiate a 6503 * preemption from blocking on an 'sleeping' spin/rwlock. 6504 */ 6505 #define SM_IDLE (-1) 6506 #define SM_NONE 0 6507 #define SM_PREEMPT 1 6508 #define SM_RTLOCK_WAIT 2 6509 6510 /* 6511 * Helper function for __schedule() 6512 * 6513 * Tries to deactivate the task, unless the should_block arg 6514 * is false or if a signal is pending. In the case a signal 6515 * is pending, marks the task's __state as RUNNING (and clear 6516 * blocked_on). 6517 */ 6518 static bool try_to_block_task(struct rq *rq, struct task_struct *p, 6519 unsigned long *task_state_p, bool should_block) 6520 { 6521 unsigned long task_state = *task_state_p; 6522 int flags = DEQUEUE_NOCLOCK; 6523 6524 if (signal_pending_state(task_state, p)) { 6525 WRITE_ONCE(p->__state, TASK_RUNNING); 6526 *task_state_p = TASK_RUNNING; 6527 return false; 6528 } 6529 6530 /* 6531 * We check should_block after signal_pending because we 6532 * will want to wake the task in that case. But if 6533 * should_block is false, its likely due to the task being 6534 * blocked on a mutex, and we want to keep it on the runqueue 6535 * to be selectable for proxy-execution. 6536 */ 6537 if (!should_block) 6538 return false; 6539 6540 p->sched_contributes_to_load = 6541 (task_state & TASK_UNINTERRUPTIBLE) && 6542 !(task_state & TASK_NOLOAD) && 6543 !(task_state & TASK_FROZEN); 6544 6545 if (unlikely(is_special_task_state(task_state))) 6546 flags |= DEQUEUE_SPECIAL; 6547 6548 /* 6549 * __schedule() ttwu() 6550 * prev_state = prev->state; if (p->on_rq && ...) 6551 * if (prev_state) goto out; 6552 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 6553 * p->state = TASK_WAKING 6554 * 6555 * Where __schedule() and ttwu() have matching control dependencies. 6556 * 6557 * After this, schedule() must not care about p->state any more. 6558 */ 6559 block_task(rq, p, flags); 6560 return true; 6561 } 6562 6563 #ifdef CONFIG_SCHED_PROXY_EXEC 6564 static inline struct task_struct *proxy_resched_idle(struct rq *rq) 6565 { 6566 put_prev_set_next_task(rq, rq->donor, rq->idle); 6567 rq_set_donor(rq, rq->idle); 6568 set_tsk_need_resched(rq->idle); 6569 return rq->idle; 6570 } 6571 6572 static bool __proxy_deactivate(struct rq *rq, struct task_struct *donor) 6573 { 6574 unsigned long state = READ_ONCE(donor->__state); 6575 6576 /* Don't deactivate if the state has been changed to TASK_RUNNING */ 6577 if (state == TASK_RUNNING) 6578 return false; 6579 /* 6580 * Because we got donor from pick_next_task(), it is *crucial* 6581 * that we call proxy_resched_idle() before we deactivate it. 6582 * As once we deactivate donor, donor->on_rq is set to zero, 6583 * which allows ttwu() to immediately try to wake the task on 6584 * another rq. So we cannot use *any* references to donor 6585 * after that point. So things like cfs_rq->curr or rq->donor 6586 * need to be changed from next *before* we deactivate. 6587 */ 6588 proxy_resched_idle(rq); 6589 return try_to_block_task(rq, donor, &state, true); 6590 } 6591 6592 static struct task_struct *proxy_deactivate(struct rq *rq, struct task_struct *donor) 6593 { 6594 if (!__proxy_deactivate(rq, donor)) { 6595 /* 6596 * XXX: For now, if deactivation failed, set donor 6597 * as unblocked, as we aren't doing proxy-migrations 6598 * yet (more logic will be needed then). 6599 */ 6600 donor->blocked_on = NULL; 6601 } 6602 return NULL; 6603 } 6604 6605 /* 6606 * Find runnable lock owner to proxy for mutex blocked donor 6607 * 6608 * Follow the blocked-on relation: 6609 * task->blocked_on -> mutex->owner -> task... 6610 * 6611 * Lock order: 6612 * 6613 * p->pi_lock 6614 * rq->lock 6615 * mutex->wait_lock 6616 * 6617 * Returns the task that is going to be used as execution context (the one 6618 * that is actually going to be run on cpu_of(rq)). 6619 */ 6620 static struct task_struct * 6621 find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) 6622 { 6623 struct task_struct *owner = NULL; 6624 int this_cpu = cpu_of(rq); 6625 struct task_struct *p; 6626 struct mutex *mutex; 6627 6628 /* Follow blocked_on chain. */ 6629 for (p = donor; task_is_blocked(p); p = owner) { 6630 mutex = p->blocked_on; 6631 /* Something changed in the chain, so pick again */ 6632 if (!mutex) 6633 return NULL; 6634 /* 6635 * By taking mutex->wait_lock we hold off concurrent mutex_unlock() 6636 * and ensure @owner sticks around. 6637 */ 6638 guard(raw_spinlock)(&mutex->wait_lock); 6639 6640 /* Check again that p is blocked with wait_lock held */ 6641 if (mutex != __get_task_blocked_on(p)) { 6642 /* 6643 * Something changed in the blocked_on chain and 6644 * we don't know if only at this level. So, let's 6645 * just bail out completely and let __schedule() 6646 * figure things out (pick_again loop). 6647 */ 6648 return NULL; 6649 } 6650 6651 owner = __mutex_owner(mutex); 6652 if (!owner) { 6653 __clear_task_blocked_on(p, mutex); 6654 return p; 6655 } 6656 6657 if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) { 6658 /* XXX Don't handle blocked owners/delayed dequeue yet */ 6659 return proxy_deactivate(rq, donor); 6660 } 6661 6662 if (task_cpu(owner) != this_cpu) { 6663 /* XXX Don't handle migrations yet */ 6664 return proxy_deactivate(rq, donor); 6665 } 6666 6667 if (task_on_rq_migrating(owner)) { 6668 /* 6669 * One of the chain of mutex owners is currently migrating to this 6670 * CPU, but has not yet been enqueued because we are holding the 6671 * rq lock. As a simple solution, just schedule rq->idle to give 6672 * the migration a chance to complete. Much like the migrate_task 6673 * case we should end up back in find_proxy_task(), this time 6674 * hopefully with all relevant tasks already enqueued. 6675 */ 6676 return proxy_resched_idle(rq); 6677 } 6678 6679 /* 6680 * Its possible to race where after we check owner->on_rq 6681 * but before we check (owner_cpu != this_cpu) that the 6682 * task on another cpu was migrated back to this cpu. In 6683 * that case it could slip by our checks. So double check 6684 * we are still on this cpu and not migrating. If we get 6685 * inconsistent results, try again. 6686 */ 6687 if (!task_on_rq_queued(owner) || task_cpu(owner) != this_cpu) 6688 return NULL; 6689 6690 if (owner == p) { 6691 /* 6692 * It's possible we interleave with mutex_unlock like: 6693 * 6694 * lock(&rq->lock); 6695 * find_proxy_task() 6696 * mutex_unlock() 6697 * lock(&wait_lock); 6698 * donor(owner) = current->blocked_donor; 6699 * unlock(&wait_lock); 6700 * 6701 * wake_up_q(); 6702 * ... 6703 * ttwu_runnable() 6704 * __task_rq_lock() 6705 * lock(&wait_lock); 6706 * owner == p 6707 * 6708 * Which leaves us to finish the ttwu_runnable() and make it go. 6709 * 6710 * So schedule rq->idle so that ttwu_runnable() can get the rq 6711 * lock and mark owner as running. 6712 */ 6713 return proxy_resched_idle(rq); 6714 } 6715 /* 6716 * OK, now we're absolutely sure @owner is on this 6717 * rq, therefore holding @rq->lock is sufficient to 6718 * guarantee its existence, as per ttwu_remote(). 6719 */ 6720 } 6721 6722 WARN_ON_ONCE(owner && !owner->on_rq); 6723 return owner; 6724 } 6725 #else /* SCHED_PROXY_EXEC */ 6726 static struct task_struct * 6727 find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) 6728 { 6729 WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n"); 6730 return donor; 6731 } 6732 #endif /* SCHED_PROXY_EXEC */ 6733 6734 static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner) 6735 { 6736 if (!sched_proxy_exec()) 6737 return; 6738 /* 6739 * pick_next_task() calls set_next_task() on the chosen task 6740 * at some point, which ensures it is not push/pullable. 6741 * However, the chosen/donor task *and* the mutex owner form an 6742 * atomic pair wrt push/pull. 6743 * 6744 * Make sure owner we run is not pushable. Unfortunately we can 6745 * only deal with that by means of a dequeue/enqueue cycle. :-/ 6746 */ 6747 dequeue_task(rq, owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE); 6748 enqueue_task(rq, owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE); 6749 } 6750 6751 /* 6752 * __schedule() is the main scheduler function. 6753 * 6754 * The main means of driving the scheduler and thus entering this function are: 6755 * 6756 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 6757 * 6758 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 6759 * paths. For example, see arch/x86/entry_64.S. 6760 * 6761 * To drive preemption between tasks, the scheduler sets the flag in timer 6762 * interrupt handler sched_tick(). 6763 * 6764 * 3. Wakeups don't really cause entry into schedule(). They add a 6765 * task to the run-queue and that's it. 6766 * 6767 * Now, if the new task added to the run-queue preempts the current 6768 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 6769 * called on the nearest possible occasion: 6770 * 6771 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 6772 * 6773 * - in syscall or exception context, at the next outmost 6774 * preempt_enable(). (this might be as soon as the wake_up()'s 6775 * spin_unlock()!) 6776 * 6777 * - in IRQ context, return from interrupt-handler to 6778 * preemptible context 6779 * 6780 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 6781 * then at the next: 6782 * 6783 * - cond_resched() call 6784 * - explicit schedule() call 6785 * - return from syscall or exception to user-space 6786 * - return from interrupt-handler to user-space 6787 * 6788 * WARNING: must be called with preemption disabled! 6789 */ 6790 static void __sched notrace __schedule(int sched_mode) 6791 { 6792 struct task_struct *prev, *next; 6793 /* 6794 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted 6795 * as a preemption by schedule_debug() and RCU. 6796 */ 6797 bool preempt = sched_mode > SM_NONE; 6798 bool is_switch = false; 6799 unsigned long *switch_count; 6800 unsigned long prev_state; 6801 struct rq_flags rf; 6802 struct rq *rq; 6803 int cpu; 6804 6805 /* Trace preemptions consistently with task switches */ 6806 trace_sched_entry_tp(sched_mode == SM_PREEMPT); 6807 6808 cpu = smp_processor_id(); 6809 rq = cpu_rq(cpu); 6810 prev = rq->curr; 6811 6812 schedule_debug(prev, preempt); 6813 6814 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6815 hrtick_clear(rq); 6816 6817 klp_sched_try_switch(prev); 6818 6819 local_irq_disable(); 6820 rcu_note_context_switch(preempt); 6821 6822 /* 6823 * Make sure that signal_pending_state()->signal_pending() below 6824 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 6825 * done by the caller to avoid the race with signal_wake_up(): 6826 * 6827 * __set_current_state(@state) signal_wake_up() 6828 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 6829 * wake_up_state(p, state) 6830 * LOCK rq->lock LOCK p->pi_state 6831 * smp_mb__after_spinlock() smp_mb__after_spinlock() 6832 * if (signal_pending_state()) if (p->state & @state) 6833 * 6834 * Also, the membarrier system call requires a full memory barrier 6835 * after coming from user-space, before storing to rq->curr; this 6836 * barrier matches a full barrier in the proximity of the membarrier 6837 * system call exit. 6838 */ 6839 rq_lock(rq, &rf); 6840 smp_mb__after_spinlock(); 6841 6842 /* Promote REQ to ACT */ 6843 rq->clock_update_flags <<= 1; 6844 update_rq_clock(rq); 6845 rq->clock_update_flags = RQCF_UPDATED; 6846 6847 switch_count = &prev->nivcsw; 6848 6849 /* Task state changes only considers SM_PREEMPT as preemption */ 6850 preempt = sched_mode == SM_PREEMPT; 6851 6852 /* 6853 * We must load prev->state once (task_struct::state is volatile), such 6854 * that we form a control dependency vs deactivate_task() below. 6855 */ 6856 prev_state = READ_ONCE(prev->__state); 6857 if (sched_mode == SM_IDLE) { 6858 /* SCX must consult the BPF scheduler to tell if rq is empty */ 6859 if (!rq->nr_running && !scx_enabled()) { 6860 next = prev; 6861 goto picked; 6862 } 6863 } else if (!preempt && prev_state) { 6864 /* 6865 * We pass task_is_blocked() as the should_block arg 6866 * in order to keep mutex-blocked tasks on the runqueue 6867 * for slection with proxy-exec (without proxy-exec 6868 * task_is_blocked() will always be false). 6869 */ 6870 try_to_block_task(rq, prev, &prev_state, 6871 !task_is_blocked(prev)); 6872 switch_count = &prev->nvcsw; 6873 } 6874 6875 pick_again: 6876 next = pick_next_task(rq, rq->donor, &rf); 6877 rq_set_donor(rq, next); 6878 if (unlikely(task_is_blocked(next))) { 6879 next = find_proxy_task(rq, next, &rf); 6880 if (!next) 6881 goto pick_again; 6882 if (next == rq->idle) 6883 goto keep_resched; 6884 } 6885 picked: 6886 clear_tsk_need_resched(prev); 6887 clear_preempt_need_resched(); 6888 keep_resched: 6889 rq->last_seen_need_resched_ns = 0; 6890 6891 is_switch = prev != next; 6892 if (likely(is_switch)) { 6893 rq->nr_switches++; 6894 /* 6895 * RCU users of rcu_dereference(rq->curr) may not see 6896 * changes to task_struct made by pick_next_task(). 6897 */ 6898 RCU_INIT_POINTER(rq->curr, next); 6899 6900 if (!task_current_donor(rq, next)) 6901 proxy_tag_curr(rq, next); 6902 6903 /* 6904 * The membarrier system call requires each architecture 6905 * to have a full memory barrier after updating 6906 * rq->curr, before returning to user-space. 6907 * 6908 * Here are the schemes providing that barrier on the 6909 * various architectures: 6910 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC, 6911 * RISC-V. switch_mm() relies on membarrier_arch_switch_mm() 6912 * on PowerPC and on RISC-V. 6913 * - finish_lock_switch() for weakly-ordered 6914 * architectures where spin_unlock is a full barrier, 6915 * - switch_to() for arm64 (weakly-ordered, spin_unlock 6916 * is a RELEASE barrier), 6917 * 6918 * The barrier matches a full barrier in the proximity of 6919 * the membarrier system call entry. 6920 * 6921 * On RISC-V, this barrier pairing is also needed for the 6922 * SYNC_CORE command when switching between processes, cf. 6923 * the inline comments in membarrier_arch_switch_mm(). 6924 */ 6925 ++*switch_count; 6926 6927 migrate_disable_switch(rq, prev); 6928 psi_account_irqtime(rq, prev, next); 6929 psi_sched_switch(prev, next, !task_on_rq_queued(prev) || 6930 prev->se.sched_delayed); 6931 6932 trace_sched_switch(preempt, prev, next, prev_state); 6933 6934 /* Also unlocks the rq: */ 6935 rq = context_switch(rq, prev, next, &rf); 6936 } else { 6937 /* In case next was already curr but just got blocked_donor */ 6938 if (!task_current_donor(rq, next)) 6939 proxy_tag_curr(rq, next); 6940 6941 rq_unpin_lock(rq, &rf); 6942 __balance_callbacks(rq); 6943 raw_spin_rq_unlock_irq(rq); 6944 } 6945 trace_sched_exit_tp(is_switch); 6946 } 6947 6948 void __noreturn do_task_dead(void) 6949 { 6950 /* Causes final put_task_struct in finish_task_switch(): */ 6951 set_special_state(TASK_DEAD); 6952 6953 /* Tell freezer to ignore us: */ 6954 current->flags |= PF_NOFREEZE; 6955 6956 __schedule(SM_NONE); 6957 BUG(); 6958 6959 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 6960 for (;;) 6961 cpu_relax(); 6962 } 6963 6964 static inline void sched_submit_work(struct task_struct *tsk) 6965 { 6966 static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG); 6967 unsigned int task_flags; 6968 6969 /* 6970 * Establish LD_WAIT_CONFIG context to ensure none of the code called 6971 * will use a blocking primitive -- which would lead to recursion. 6972 */ 6973 lock_map_acquire_try(&sched_map); 6974 6975 task_flags = tsk->flags; 6976 /* 6977 * If a worker goes to sleep, notify and ask workqueue whether it 6978 * wants to wake up a task to maintain concurrency. 6979 */ 6980 if (task_flags & PF_WQ_WORKER) 6981 wq_worker_sleeping(tsk); 6982 else if (task_flags & PF_IO_WORKER) 6983 io_wq_worker_sleeping(tsk); 6984 6985 /* 6986 * spinlock and rwlock must not flush block requests. This will 6987 * deadlock if the callback attempts to acquire a lock which is 6988 * already acquired. 6989 */ 6990 WARN_ON_ONCE(current->__state & TASK_RTLOCK_WAIT); 6991 6992 /* 6993 * If we are going to sleep and we have plugged IO queued, 6994 * make sure to submit it to avoid deadlocks. 6995 */ 6996 blk_flush_plug(tsk->plug, true); 6997 6998 lock_map_release(&sched_map); 6999 } 7000 7001 static void sched_update_worker(struct task_struct *tsk) 7002 { 7003 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { 7004 if (tsk->flags & PF_BLOCK_TS) 7005 blk_plug_invalidate_ts(tsk); 7006 if (tsk->flags & PF_WQ_WORKER) 7007 wq_worker_running(tsk); 7008 else if (tsk->flags & PF_IO_WORKER) 7009 io_wq_worker_running(tsk); 7010 } 7011 } 7012 7013 static __always_inline void __schedule_loop(int sched_mode) 7014 { 7015 do { 7016 preempt_disable(); 7017 __schedule(sched_mode); 7018 sched_preempt_enable_no_resched(); 7019 } while (need_resched()); 7020 } 7021 7022 asmlinkage __visible void __sched schedule(void) 7023 { 7024 struct task_struct *tsk = current; 7025 7026 #ifdef CONFIG_RT_MUTEXES 7027 lockdep_assert(!tsk->sched_rt_mutex); 7028 #endif 7029 7030 if (!task_is_running(tsk)) 7031 sched_submit_work(tsk); 7032 __schedule_loop(SM_NONE); 7033 sched_update_worker(tsk); 7034 } 7035 EXPORT_SYMBOL(schedule); 7036 7037 /* 7038 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 7039 * state (have scheduled out non-voluntarily) by making sure that all 7040 * tasks have either left the run queue or have gone into user space. 7041 * As idle tasks do not do either, they must not ever be preempted 7042 * (schedule out non-voluntarily). 7043 * 7044 * schedule_idle() is similar to schedule_preempt_disable() except that it 7045 * never enables preemption because it does not call sched_submit_work(). 7046 */ 7047 void __sched schedule_idle(void) 7048 { 7049 /* 7050 * As this skips calling sched_submit_work(), which the idle task does 7051 * regardless because that function is a NOP when the task is in a 7052 * TASK_RUNNING state, make sure this isn't used someplace that the 7053 * current task can be in any other state. Note, idle is always in the 7054 * TASK_RUNNING state. 7055 */ 7056 WARN_ON_ONCE(current->__state); 7057 do { 7058 __schedule(SM_IDLE); 7059 } while (need_resched()); 7060 } 7061 7062 #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) 7063 asmlinkage __visible void __sched schedule_user(void) 7064 { 7065 /* 7066 * If we come here after a random call to set_need_resched(), 7067 * or we have been woken up remotely but the IPI has not yet arrived, 7068 * we haven't yet exited the RCU idle mode. Do it here manually until 7069 * we find a better solution. 7070 * 7071 * NB: There are buggy callers of this function. Ideally we 7072 * should warn if prev_state != CT_STATE_USER, but that will trigger 7073 * too frequently to make sense yet. 7074 */ 7075 enum ctx_state prev_state = exception_enter(); 7076 schedule(); 7077 exception_exit(prev_state); 7078 } 7079 #endif 7080 7081 /** 7082 * schedule_preempt_disabled - called with preemption disabled 7083 * 7084 * Returns with preemption disabled. Note: preempt_count must be 1 7085 */ 7086 void __sched schedule_preempt_disabled(void) 7087 { 7088 sched_preempt_enable_no_resched(); 7089 schedule(); 7090 preempt_disable(); 7091 } 7092 7093 #ifdef CONFIG_PREEMPT_RT 7094 void __sched notrace schedule_rtlock(void) 7095 { 7096 __schedule_loop(SM_RTLOCK_WAIT); 7097 } 7098 NOKPROBE_SYMBOL(schedule_rtlock); 7099 #endif 7100 7101 static void __sched notrace preempt_schedule_common(void) 7102 { 7103 do { 7104 /* 7105 * Because the function tracer can trace preempt_count_sub() 7106 * and it also uses preempt_enable/disable_notrace(), if 7107 * NEED_RESCHED is set, the preempt_enable_notrace() called 7108 * by the function tracer will call this function again and 7109 * cause infinite recursion. 7110 * 7111 * Preemption must be disabled here before the function 7112 * tracer can trace. Break up preempt_disable() into two 7113 * calls. One to disable preemption without fear of being 7114 * traced. The other to still record the preemption latency, 7115 * which can also be traced by the function tracer. 7116 */ 7117 preempt_disable_notrace(); 7118 preempt_latency_start(1); 7119 __schedule(SM_PREEMPT); 7120 preempt_latency_stop(1); 7121 preempt_enable_no_resched_notrace(); 7122 7123 /* 7124 * Check again in case we missed a preemption opportunity 7125 * between schedule and now. 7126 */ 7127 } while (need_resched()); 7128 } 7129 7130 #ifdef CONFIG_PREEMPTION 7131 /* 7132 * This is the entry point to schedule() from in-kernel preemption 7133 * off of preempt_enable. 7134 */ 7135 asmlinkage __visible void __sched notrace preempt_schedule(void) 7136 { 7137 /* 7138 * If there is a non-zero preempt_count or interrupts are disabled, 7139 * we do not want to preempt the current task. Just return.. 7140 */ 7141 if (likely(!preemptible())) 7142 return; 7143 preempt_schedule_common(); 7144 } 7145 NOKPROBE_SYMBOL(preempt_schedule); 7146 EXPORT_SYMBOL(preempt_schedule); 7147 7148 #ifdef CONFIG_PREEMPT_DYNAMIC 7149 # ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7150 # ifndef preempt_schedule_dynamic_enabled 7151 # define preempt_schedule_dynamic_enabled preempt_schedule 7152 # define preempt_schedule_dynamic_disabled NULL 7153 # endif 7154 DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 7155 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 7156 # elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7157 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 7158 void __sched notrace dynamic_preempt_schedule(void) 7159 { 7160 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 7161 return; 7162 preempt_schedule(); 7163 } 7164 NOKPROBE_SYMBOL(dynamic_preempt_schedule); 7165 EXPORT_SYMBOL(dynamic_preempt_schedule); 7166 # endif 7167 #endif /* CONFIG_PREEMPT_DYNAMIC */ 7168 7169 /** 7170 * preempt_schedule_notrace - preempt_schedule called by tracing 7171 * 7172 * The tracing infrastructure uses preempt_enable_notrace to prevent 7173 * recursion and tracing preempt enabling caused by the tracing 7174 * infrastructure itself. But as tracing can happen in areas coming 7175 * from userspace or just about to enter userspace, a preempt enable 7176 * can occur before user_exit() is called. This will cause the scheduler 7177 * to be called when the system is still in usermode. 7178 * 7179 * To prevent this, the preempt_enable_notrace will use this function 7180 * instead of preempt_schedule() to exit user context if needed before 7181 * calling the scheduler. 7182 */ 7183 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 7184 { 7185 enum ctx_state prev_ctx; 7186 7187 if (likely(!preemptible())) 7188 return; 7189 7190 do { 7191 /* 7192 * Because the function tracer can trace preempt_count_sub() 7193 * and it also uses preempt_enable/disable_notrace(), if 7194 * NEED_RESCHED is set, the preempt_enable_notrace() called 7195 * by the function tracer will call this function again and 7196 * cause infinite recursion. 7197 * 7198 * Preemption must be disabled here before the function 7199 * tracer can trace. Break up preempt_disable() into two 7200 * calls. One to disable preemption without fear of being 7201 * traced. The other to still record the preemption latency, 7202 * which can also be traced by the function tracer. 7203 */ 7204 preempt_disable_notrace(); 7205 preempt_latency_start(1); 7206 /* 7207 * Needs preempt disabled in case user_exit() is traced 7208 * and the tracer calls preempt_enable_notrace() causing 7209 * an infinite recursion. 7210 */ 7211 prev_ctx = exception_enter(); 7212 __schedule(SM_PREEMPT); 7213 exception_exit(prev_ctx); 7214 7215 preempt_latency_stop(1); 7216 preempt_enable_no_resched_notrace(); 7217 } while (need_resched()); 7218 } 7219 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 7220 7221 #ifdef CONFIG_PREEMPT_DYNAMIC 7222 # if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7223 # ifndef preempt_schedule_notrace_dynamic_enabled 7224 # define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 7225 # define preempt_schedule_notrace_dynamic_disabled NULL 7226 # endif 7227 DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 7228 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 7229 # elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7230 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 7231 void __sched notrace dynamic_preempt_schedule_notrace(void) 7232 { 7233 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 7234 return; 7235 preempt_schedule_notrace(); 7236 } 7237 NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 7238 EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 7239 # endif 7240 #endif 7241 7242 #endif /* CONFIG_PREEMPTION */ 7243 7244 /* 7245 * This is the entry point to schedule() from kernel preemption 7246 * off of IRQ context. 7247 * Note, that this is called and return with IRQs disabled. This will 7248 * protect us against recursive calling from IRQ contexts. 7249 */ 7250 asmlinkage __visible void __sched preempt_schedule_irq(void) 7251 { 7252 enum ctx_state prev_state; 7253 7254 /* Catch callers which need to be fixed */ 7255 BUG_ON(preempt_count() || !irqs_disabled()); 7256 7257 prev_state = exception_enter(); 7258 7259 do { 7260 preempt_disable(); 7261 local_irq_enable(); 7262 __schedule(SM_PREEMPT); 7263 local_irq_disable(); 7264 sched_preempt_enable_no_resched(); 7265 } while (need_resched()); 7266 7267 exception_exit(prev_state); 7268 } 7269 7270 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 7271 void *key) 7272 { 7273 WARN_ON_ONCE(wake_flags & ~(WF_SYNC|WF_CURRENT_CPU)); 7274 return try_to_wake_up(curr->private, mode, wake_flags); 7275 } 7276 EXPORT_SYMBOL(default_wake_function); 7277 7278 const struct sched_class *__setscheduler_class(int policy, int prio) 7279 { 7280 if (dl_prio(prio)) 7281 return &dl_sched_class; 7282 7283 if (rt_prio(prio)) 7284 return &rt_sched_class; 7285 7286 #ifdef CONFIG_SCHED_CLASS_EXT 7287 if (task_should_scx(policy)) 7288 return &ext_sched_class; 7289 #endif 7290 7291 return &fair_sched_class; 7292 } 7293 7294 #ifdef CONFIG_RT_MUTEXES 7295 7296 /* 7297 * Would be more useful with typeof()/auto_type but they don't mix with 7298 * bit-fields. Since it's a local thing, use int. Keep the generic sounding 7299 * name such that if someone were to implement this function we get to compare 7300 * notes. 7301 */ 7302 #define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; }) 7303 7304 void rt_mutex_pre_schedule(void) 7305 { 7306 lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); 7307 sched_submit_work(current); 7308 } 7309 7310 void rt_mutex_schedule(void) 7311 { 7312 lockdep_assert(current->sched_rt_mutex); 7313 __schedule_loop(SM_NONE); 7314 } 7315 7316 void rt_mutex_post_schedule(void) 7317 { 7318 sched_update_worker(current); 7319 lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); 7320 } 7321 7322 /* 7323 * rt_mutex_setprio - set the current priority of a task 7324 * @p: task to boost 7325 * @pi_task: donor task 7326 * 7327 * This function changes the 'effective' priority of a task. It does 7328 * not touch ->normal_prio like __setscheduler(). 7329 * 7330 * Used by the rt_mutex code to implement priority inheritance 7331 * logic. Call site only calls if the priority of the task changed. 7332 */ 7333 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 7334 { 7335 int prio, oldprio, queue_flag = 7336 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7337 const struct sched_class *prev_class, *next_class; 7338 struct rq_flags rf; 7339 struct rq *rq; 7340 7341 /* XXX used to be waiter->prio, not waiter->task->prio */ 7342 prio = __rt_effective_prio(pi_task, p->normal_prio); 7343 7344 /* 7345 * If nothing changed; bail early. 7346 */ 7347 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 7348 return; 7349 7350 rq = __task_rq_lock(p, &rf); 7351 update_rq_clock(rq); 7352 /* 7353 * Set under pi_lock && rq->lock, such that the value can be used under 7354 * either lock. 7355 * 7356 * Note that there is loads of tricky to make this pointer cache work 7357 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 7358 * ensure a task is de-boosted (pi_task is set to NULL) before the 7359 * task is allowed to run again (and can exit). This ensures the pointer 7360 * points to a blocked task -- which guarantees the task is present. 7361 */ 7362 p->pi_top_task = pi_task; 7363 7364 /* 7365 * For FIFO/RR we only need to set prio, if that matches we're done. 7366 */ 7367 if (prio == p->prio && !dl_prio(prio)) 7368 goto out_unlock; 7369 7370 /* 7371 * Idle task boosting is a no-no in general. There is one 7372 * exception, when PREEMPT_RT and NOHZ is active: 7373 * 7374 * The idle task calls get_next_timer_interrupt() and holds 7375 * the timer wheel base->lock on the CPU and another CPU wants 7376 * to access the timer (probably to cancel it). We can safely 7377 * ignore the boosting request, as the idle CPU runs this code 7378 * with interrupts disabled and will complete the lock 7379 * protected section without being interrupted. So there is no 7380 * real need to boost. 7381 */ 7382 if (unlikely(p == rq->idle)) { 7383 WARN_ON(p != rq->curr); 7384 WARN_ON(p->pi_blocked_on); 7385 goto out_unlock; 7386 } 7387 7388 trace_sched_pi_setprio(p, pi_task); 7389 oldprio = p->prio; 7390 7391 if (oldprio == prio && !dl_prio(prio)) 7392 queue_flag &= ~DEQUEUE_MOVE; 7393 7394 prev_class = p->sched_class; 7395 next_class = __setscheduler_class(p->policy, prio); 7396 7397 if (prev_class != next_class && p->se.sched_delayed) 7398 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED | DEQUEUE_NOCLOCK); 7399 7400 scoped_guard (sched_change, p, queue_flag) { 7401 /* 7402 * Boosting condition are: 7403 * 1. -rt task is running and holds mutex A 7404 * --> -dl task blocks on mutex A 7405 * 7406 * 2. -dl task is running and holds mutex A 7407 * --> -dl task blocks on mutex A and could preempt the 7408 * running task 7409 */ 7410 if (dl_prio(prio)) { 7411 if (!dl_prio(p->normal_prio) || 7412 (pi_task && dl_prio(pi_task->prio) && 7413 dl_entity_preempt(&pi_task->dl, &p->dl))) { 7414 p->dl.pi_se = pi_task->dl.pi_se; 7415 scope->flags |= ENQUEUE_REPLENISH; 7416 } else { 7417 p->dl.pi_se = &p->dl; 7418 } 7419 } else if (rt_prio(prio)) { 7420 if (dl_prio(oldprio)) 7421 p->dl.pi_se = &p->dl; 7422 if (oldprio < prio) 7423 scope->flags |= ENQUEUE_HEAD; 7424 } else { 7425 if (dl_prio(oldprio)) 7426 p->dl.pi_se = &p->dl; 7427 if (rt_prio(oldprio)) 7428 p->rt.timeout = 0; 7429 } 7430 7431 p->sched_class = next_class; 7432 p->prio = prio; 7433 7434 check_class_changing(rq, p, prev_class); 7435 } 7436 7437 check_class_changed(rq, p, prev_class, oldprio); 7438 out_unlock: 7439 /* Avoid rq from going away on us: */ 7440 preempt_disable(); 7441 7442 rq_unpin_lock(rq, &rf); 7443 __balance_callbacks(rq); 7444 raw_spin_rq_unlock(rq); 7445 7446 preempt_enable(); 7447 } 7448 #endif /* CONFIG_RT_MUTEXES */ 7449 7450 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 7451 int __sched __cond_resched(void) 7452 { 7453 if (should_resched(0) && !irqs_disabled()) { 7454 preempt_schedule_common(); 7455 return 1; 7456 } 7457 /* 7458 * In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick 7459 * whether the current CPU is in an RCU read-side critical section, 7460 * so the tick can report quiescent states even for CPUs looping 7461 * in kernel context. In contrast, in non-preemptible kernels, 7462 * RCU readers leave no in-memory hints, which means that CPU-bound 7463 * processes executing in kernel context might never report an 7464 * RCU quiescent state. Therefore, the following code causes 7465 * cond_resched() to report a quiescent state, but only when RCU 7466 * is in urgent need of one. 7467 * A third case, preemptible, but non-PREEMPT_RCU provides for 7468 * urgently needed quiescent states via rcu_flavor_sched_clock_irq(). 7469 */ 7470 #ifndef CONFIG_PREEMPT_RCU 7471 rcu_all_qs(); 7472 #endif 7473 return 0; 7474 } 7475 EXPORT_SYMBOL(__cond_resched); 7476 #endif 7477 7478 #ifdef CONFIG_PREEMPT_DYNAMIC 7479 # ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7480 # define cond_resched_dynamic_enabled __cond_resched 7481 # define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 7482 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 7483 EXPORT_STATIC_CALL_TRAMP(cond_resched); 7484 7485 # define might_resched_dynamic_enabled __cond_resched 7486 # define might_resched_dynamic_disabled ((void *)&__static_call_return0) 7487 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 7488 EXPORT_STATIC_CALL_TRAMP(might_resched); 7489 # elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7490 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 7491 int __sched dynamic_cond_resched(void) 7492 { 7493 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 7494 return 0; 7495 return __cond_resched(); 7496 } 7497 EXPORT_SYMBOL(dynamic_cond_resched); 7498 7499 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 7500 int __sched dynamic_might_resched(void) 7501 { 7502 if (!static_branch_unlikely(&sk_dynamic_might_resched)) 7503 return 0; 7504 return __cond_resched(); 7505 } 7506 EXPORT_SYMBOL(dynamic_might_resched); 7507 # endif 7508 #endif /* CONFIG_PREEMPT_DYNAMIC */ 7509 7510 /* 7511 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 7512 * call schedule, and on return reacquire the lock. 7513 * 7514 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 7515 * operations here to prevent schedule() from being called twice (once via 7516 * spin_unlock(), once by hand). 7517 */ 7518 int __cond_resched_lock(spinlock_t *lock) 7519 { 7520 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7521 int ret = 0; 7522 7523 lockdep_assert_held(lock); 7524 7525 if (spin_needbreak(lock) || resched) { 7526 spin_unlock(lock); 7527 if (!_cond_resched()) 7528 cpu_relax(); 7529 ret = 1; 7530 spin_lock(lock); 7531 } 7532 return ret; 7533 } 7534 EXPORT_SYMBOL(__cond_resched_lock); 7535 7536 int __cond_resched_rwlock_read(rwlock_t *lock) 7537 { 7538 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7539 int ret = 0; 7540 7541 lockdep_assert_held_read(lock); 7542 7543 if (rwlock_needbreak(lock) || resched) { 7544 read_unlock(lock); 7545 if (!_cond_resched()) 7546 cpu_relax(); 7547 ret = 1; 7548 read_lock(lock); 7549 } 7550 return ret; 7551 } 7552 EXPORT_SYMBOL(__cond_resched_rwlock_read); 7553 7554 int __cond_resched_rwlock_write(rwlock_t *lock) 7555 { 7556 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7557 int ret = 0; 7558 7559 lockdep_assert_held_write(lock); 7560 7561 if (rwlock_needbreak(lock) || resched) { 7562 write_unlock(lock); 7563 if (!_cond_resched()) 7564 cpu_relax(); 7565 ret = 1; 7566 write_lock(lock); 7567 } 7568 return ret; 7569 } 7570 EXPORT_SYMBOL(__cond_resched_rwlock_write); 7571 7572 #ifdef CONFIG_PREEMPT_DYNAMIC 7573 7574 # ifdef CONFIG_GENERIC_IRQ_ENTRY 7575 # include <linux/irq-entry-common.h> 7576 # endif 7577 7578 /* 7579 * SC:cond_resched 7580 * SC:might_resched 7581 * SC:preempt_schedule 7582 * SC:preempt_schedule_notrace 7583 * SC:irqentry_exit_cond_resched 7584 * 7585 * 7586 * NONE: 7587 * cond_resched <- __cond_resched 7588 * might_resched <- RET0 7589 * preempt_schedule <- NOP 7590 * preempt_schedule_notrace <- NOP 7591 * irqentry_exit_cond_resched <- NOP 7592 * dynamic_preempt_lazy <- false 7593 * 7594 * VOLUNTARY: 7595 * cond_resched <- __cond_resched 7596 * might_resched <- __cond_resched 7597 * preempt_schedule <- NOP 7598 * preempt_schedule_notrace <- NOP 7599 * irqentry_exit_cond_resched <- NOP 7600 * dynamic_preempt_lazy <- false 7601 * 7602 * FULL: 7603 * cond_resched <- RET0 7604 * might_resched <- RET0 7605 * preempt_schedule <- preempt_schedule 7606 * preempt_schedule_notrace <- preempt_schedule_notrace 7607 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 7608 * dynamic_preempt_lazy <- false 7609 * 7610 * LAZY: 7611 * cond_resched <- RET0 7612 * might_resched <- RET0 7613 * preempt_schedule <- preempt_schedule 7614 * preempt_schedule_notrace <- preempt_schedule_notrace 7615 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 7616 * dynamic_preempt_lazy <- true 7617 */ 7618 7619 enum { 7620 preempt_dynamic_undefined = -1, 7621 preempt_dynamic_none, 7622 preempt_dynamic_voluntary, 7623 preempt_dynamic_full, 7624 preempt_dynamic_lazy, 7625 }; 7626 7627 int preempt_dynamic_mode = preempt_dynamic_undefined; 7628 7629 int sched_dynamic_mode(const char *str) 7630 { 7631 # ifndef CONFIG_PREEMPT_RT 7632 if (!strcmp(str, "none")) 7633 return preempt_dynamic_none; 7634 7635 if (!strcmp(str, "voluntary")) 7636 return preempt_dynamic_voluntary; 7637 # endif 7638 7639 if (!strcmp(str, "full")) 7640 return preempt_dynamic_full; 7641 7642 # ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY 7643 if (!strcmp(str, "lazy")) 7644 return preempt_dynamic_lazy; 7645 # endif 7646 7647 return -EINVAL; 7648 } 7649 7650 # define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) 7651 # define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) 7652 7653 # if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7654 # define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 7655 # define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 7656 # elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7657 # define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) 7658 # define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) 7659 # else 7660 # error "Unsupported PREEMPT_DYNAMIC mechanism" 7661 # endif 7662 7663 static DEFINE_MUTEX(sched_dynamic_mutex); 7664 7665 static void __sched_dynamic_update(int mode) 7666 { 7667 /* 7668 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 7669 * the ZERO state, which is invalid. 7670 */ 7671 preempt_dynamic_enable(cond_resched); 7672 preempt_dynamic_enable(might_resched); 7673 preempt_dynamic_enable(preempt_schedule); 7674 preempt_dynamic_enable(preempt_schedule_notrace); 7675 preempt_dynamic_enable(irqentry_exit_cond_resched); 7676 preempt_dynamic_key_disable(preempt_lazy); 7677 7678 switch (mode) { 7679 case preempt_dynamic_none: 7680 preempt_dynamic_enable(cond_resched); 7681 preempt_dynamic_disable(might_resched); 7682 preempt_dynamic_disable(preempt_schedule); 7683 preempt_dynamic_disable(preempt_schedule_notrace); 7684 preempt_dynamic_disable(irqentry_exit_cond_resched); 7685 preempt_dynamic_key_disable(preempt_lazy); 7686 if (mode != preempt_dynamic_mode) 7687 pr_info("Dynamic Preempt: none\n"); 7688 break; 7689 7690 case preempt_dynamic_voluntary: 7691 preempt_dynamic_enable(cond_resched); 7692 preempt_dynamic_enable(might_resched); 7693 preempt_dynamic_disable(preempt_schedule); 7694 preempt_dynamic_disable(preempt_schedule_notrace); 7695 preempt_dynamic_disable(irqentry_exit_cond_resched); 7696 preempt_dynamic_key_disable(preempt_lazy); 7697 if (mode != preempt_dynamic_mode) 7698 pr_info("Dynamic Preempt: voluntary\n"); 7699 break; 7700 7701 case preempt_dynamic_full: 7702 preempt_dynamic_disable(cond_resched); 7703 preempt_dynamic_disable(might_resched); 7704 preempt_dynamic_enable(preempt_schedule); 7705 preempt_dynamic_enable(preempt_schedule_notrace); 7706 preempt_dynamic_enable(irqentry_exit_cond_resched); 7707 preempt_dynamic_key_disable(preempt_lazy); 7708 if (mode != preempt_dynamic_mode) 7709 pr_info("Dynamic Preempt: full\n"); 7710 break; 7711 7712 case preempt_dynamic_lazy: 7713 preempt_dynamic_disable(cond_resched); 7714 preempt_dynamic_disable(might_resched); 7715 preempt_dynamic_enable(preempt_schedule); 7716 preempt_dynamic_enable(preempt_schedule_notrace); 7717 preempt_dynamic_enable(irqentry_exit_cond_resched); 7718 preempt_dynamic_key_enable(preempt_lazy); 7719 if (mode != preempt_dynamic_mode) 7720 pr_info("Dynamic Preempt: lazy\n"); 7721 break; 7722 } 7723 7724 preempt_dynamic_mode = mode; 7725 } 7726 7727 void sched_dynamic_update(int mode) 7728 { 7729 mutex_lock(&sched_dynamic_mutex); 7730 __sched_dynamic_update(mode); 7731 mutex_unlock(&sched_dynamic_mutex); 7732 } 7733 7734 static int __init setup_preempt_mode(char *str) 7735 { 7736 int mode = sched_dynamic_mode(str); 7737 if (mode < 0) { 7738 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 7739 return 0; 7740 } 7741 7742 sched_dynamic_update(mode); 7743 return 1; 7744 } 7745 __setup("preempt=", setup_preempt_mode); 7746 7747 static void __init preempt_dynamic_init(void) 7748 { 7749 if (preempt_dynamic_mode == preempt_dynamic_undefined) { 7750 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { 7751 sched_dynamic_update(preempt_dynamic_none); 7752 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { 7753 sched_dynamic_update(preempt_dynamic_voluntary); 7754 } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { 7755 sched_dynamic_update(preempt_dynamic_lazy); 7756 } else { 7757 /* Default static call setting, nothing to do */ 7758 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); 7759 preempt_dynamic_mode = preempt_dynamic_full; 7760 pr_info("Dynamic Preempt: full\n"); 7761 } 7762 } 7763 } 7764 7765 # define PREEMPT_MODEL_ACCESSOR(mode) \ 7766 bool preempt_model_##mode(void) \ 7767 { \ 7768 WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ 7769 return preempt_dynamic_mode == preempt_dynamic_##mode; \ 7770 } \ 7771 EXPORT_SYMBOL_GPL(preempt_model_##mode) 7772 7773 PREEMPT_MODEL_ACCESSOR(none); 7774 PREEMPT_MODEL_ACCESSOR(voluntary); 7775 PREEMPT_MODEL_ACCESSOR(full); 7776 PREEMPT_MODEL_ACCESSOR(lazy); 7777 7778 #else /* !CONFIG_PREEMPT_DYNAMIC: */ 7779 7780 #define preempt_dynamic_mode -1 7781 7782 static inline void preempt_dynamic_init(void) { } 7783 7784 #endif /* CONFIG_PREEMPT_DYNAMIC */ 7785 7786 const char *preempt_modes[] = { 7787 "none", "voluntary", "full", "lazy", NULL, 7788 }; 7789 7790 const char *preempt_model_str(void) 7791 { 7792 bool brace = IS_ENABLED(CONFIG_PREEMPT_RT) && 7793 (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC) || 7794 IS_ENABLED(CONFIG_PREEMPT_LAZY)); 7795 static char buf[128]; 7796 7797 if (IS_ENABLED(CONFIG_PREEMPT_BUILD)) { 7798 struct seq_buf s; 7799 7800 seq_buf_init(&s, buf, sizeof(buf)); 7801 seq_buf_puts(&s, "PREEMPT"); 7802 7803 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 7804 seq_buf_printf(&s, "%sRT%s", 7805 brace ? "_{" : "_", 7806 brace ? "," : ""); 7807 7808 if (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC)) { 7809 seq_buf_printf(&s, "(%s)%s", 7810 preempt_dynamic_mode >= 0 ? 7811 preempt_modes[preempt_dynamic_mode] : "undef", 7812 brace ? "}" : ""); 7813 return seq_buf_str(&s); 7814 } 7815 7816 if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { 7817 seq_buf_printf(&s, "LAZY%s", 7818 brace ? "}" : ""); 7819 return seq_buf_str(&s); 7820 } 7821 7822 return seq_buf_str(&s); 7823 } 7824 7825 if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BUILD)) 7826 return "VOLUNTARY"; 7827 7828 return "NONE"; 7829 } 7830 7831 int io_schedule_prepare(void) 7832 { 7833 int old_iowait = current->in_iowait; 7834 7835 current->in_iowait = 1; 7836 blk_flush_plug(current->plug, true); 7837 return old_iowait; 7838 } 7839 7840 void io_schedule_finish(int token) 7841 { 7842 current->in_iowait = token; 7843 } 7844 7845 /* 7846 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 7847 * that process accounting knows that this is a task in IO wait state. 7848 */ 7849 long __sched io_schedule_timeout(long timeout) 7850 { 7851 int token; 7852 long ret; 7853 7854 token = io_schedule_prepare(); 7855 ret = schedule_timeout(timeout); 7856 io_schedule_finish(token); 7857 7858 return ret; 7859 } 7860 EXPORT_SYMBOL(io_schedule_timeout); 7861 7862 void __sched io_schedule(void) 7863 { 7864 int token; 7865 7866 token = io_schedule_prepare(); 7867 schedule(); 7868 io_schedule_finish(token); 7869 } 7870 EXPORT_SYMBOL(io_schedule); 7871 7872 void sched_show_task(struct task_struct *p) 7873 { 7874 unsigned long free; 7875 int ppid; 7876 7877 if (!try_get_task_stack(p)) 7878 return; 7879 7880 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 7881 7882 if (task_is_running(p)) 7883 pr_cont(" running task "); 7884 free = stack_not_used(p); 7885 ppid = 0; 7886 rcu_read_lock(); 7887 if (pid_alive(p)) 7888 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 7889 rcu_read_unlock(); 7890 pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d task_flags:0x%04x flags:0x%08lx\n", 7891 free, task_pid_nr(p), task_tgid_nr(p), 7892 ppid, p->flags, read_task_thread_flags(p)); 7893 7894 print_worker_info(KERN_INFO, p); 7895 print_stop_info(KERN_INFO, p); 7896 print_scx_info(KERN_INFO, p); 7897 show_stack(p, NULL, KERN_INFO); 7898 put_task_stack(p); 7899 } 7900 EXPORT_SYMBOL_GPL(sched_show_task); 7901 7902 static inline bool 7903 state_filter_match(unsigned long state_filter, struct task_struct *p) 7904 { 7905 unsigned int state = READ_ONCE(p->__state); 7906 7907 /* no filter, everything matches */ 7908 if (!state_filter) 7909 return true; 7910 7911 /* filter, but doesn't match */ 7912 if (!(state & state_filter)) 7913 return false; 7914 7915 /* 7916 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 7917 * TASK_KILLABLE). 7918 */ 7919 if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) 7920 return false; 7921 7922 return true; 7923 } 7924 7925 7926 void show_state_filter(unsigned int state_filter) 7927 { 7928 struct task_struct *g, *p; 7929 7930 rcu_read_lock(); 7931 for_each_process_thread(g, p) { 7932 /* 7933 * reset the NMI-timeout, listing all files on a slow 7934 * console might take a lot of time: 7935 * Also, reset softlockup watchdogs on all CPUs, because 7936 * another CPU might be blocked waiting for us to process 7937 * an IPI. 7938 */ 7939 touch_nmi_watchdog(); 7940 touch_all_softlockup_watchdogs(); 7941 if (state_filter_match(state_filter, p)) 7942 sched_show_task(p); 7943 } 7944 7945 if (!state_filter) 7946 sysrq_sched_debug_show(); 7947 7948 rcu_read_unlock(); 7949 /* 7950 * Only show locks if all tasks are dumped: 7951 */ 7952 if (!state_filter) 7953 debug_show_all_locks(); 7954 } 7955 7956 /** 7957 * init_idle - set up an idle thread for a given CPU 7958 * @idle: task in question 7959 * @cpu: CPU the idle task belongs to 7960 * 7961 * NOTE: this function does not set the idle thread's NEED_RESCHED 7962 * flag, to make booting more robust. 7963 */ 7964 void __init init_idle(struct task_struct *idle, int cpu) 7965 { 7966 struct affinity_context ac = (struct affinity_context) { 7967 .new_mask = cpumask_of(cpu), 7968 .flags = 0, 7969 }; 7970 struct rq *rq = cpu_rq(cpu); 7971 unsigned long flags; 7972 7973 raw_spin_lock_irqsave(&idle->pi_lock, flags); 7974 raw_spin_rq_lock(rq); 7975 7976 idle->__state = TASK_RUNNING; 7977 idle->se.exec_start = sched_clock(); 7978 /* 7979 * PF_KTHREAD should already be set at this point; regardless, make it 7980 * look like a proper per-CPU kthread. 7981 */ 7982 idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY; 7983 kthread_set_per_cpu(idle, cpu); 7984 7985 /* 7986 * No validation and serialization required at boot time and for 7987 * setting up the idle tasks of not yet online CPUs. 7988 */ 7989 set_cpus_allowed_common(idle, &ac); 7990 /* 7991 * We're having a chicken and egg problem, even though we are 7992 * holding rq->lock, the CPU isn't yet set to this CPU so the 7993 * lockdep check in task_group() will fail. 7994 * 7995 * Similar case to sched_fork(). / Alternatively we could 7996 * use task_rq_lock() here and obtain the other rq->lock. 7997 * 7998 * Silence PROVE_RCU 7999 */ 8000 rcu_read_lock(); 8001 __set_task_cpu(idle, cpu); 8002 rcu_read_unlock(); 8003 8004 rq->idle = idle; 8005 rq_set_donor(rq, idle); 8006 rcu_assign_pointer(rq->curr, idle); 8007 idle->on_rq = TASK_ON_RQ_QUEUED; 8008 idle->on_cpu = 1; 8009 raw_spin_rq_unlock(rq); 8010 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 8011 8012 /* Set the preempt count _outside_ the spinlocks! */ 8013 init_idle_preempt_count(idle, cpu); 8014 8015 /* 8016 * The idle tasks have their own, simple scheduling class: 8017 */ 8018 idle->sched_class = &idle_sched_class; 8019 ftrace_graph_init_idle_task(idle, cpu); 8020 vtime_init_idle(idle, cpu); 8021 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 8022 } 8023 8024 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 8025 const struct cpumask *trial) 8026 { 8027 int ret = 1; 8028 8029 if (cpumask_empty(cur)) 8030 return ret; 8031 8032 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 8033 8034 return ret; 8035 } 8036 8037 int task_can_attach(struct task_struct *p) 8038 { 8039 int ret = 0; 8040 8041 /* 8042 * Kthreads which disallow setaffinity shouldn't be moved 8043 * to a new cpuset; we don't want to change their CPU 8044 * affinity and isolating such threads by their set of 8045 * allowed nodes is unnecessary. Thus, cpusets are not 8046 * applicable for such threads. This prevents checking for 8047 * success of set_cpus_allowed_ptr() on all attached tasks 8048 * before cpus_mask may be changed. 8049 */ 8050 if (p->flags & PF_NO_SETAFFINITY) 8051 ret = -EINVAL; 8052 8053 return ret; 8054 } 8055 8056 bool sched_smp_initialized __read_mostly; 8057 8058 #ifdef CONFIG_NUMA_BALANCING 8059 /* Migrate current task p to target_cpu */ 8060 int migrate_task_to(struct task_struct *p, int target_cpu) 8061 { 8062 struct migration_arg arg = { p, target_cpu }; 8063 int curr_cpu = task_cpu(p); 8064 8065 if (curr_cpu == target_cpu) 8066 return 0; 8067 8068 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 8069 return -EINVAL; 8070 8071 /* TODO: This is not properly updating schedstats */ 8072 8073 trace_sched_move_numa(p, curr_cpu, target_cpu); 8074 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 8075 } 8076 8077 /* 8078 * Requeue a task on a given node and accurately track the number of NUMA 8079 * tasks on the runqueues 8080 */ 8081 void sched_setnuma(struct task_struct *p, int nid) 8082 { 8083 guard(task_rq_lock)(p); 8084 scoped_guard (sched_change, p, DEQUEUE_SAVE) 8085 p->numa_preferred_nid = nid; 8086 } 8087 #endif /* CONFIG_NUMA_BALANCING */ 8088 8089 #ifdef CONFIG_HOTPLUG_CPU 8090 /* 8091 * Invoked on the outgoing CPU in context of the CPU hotplug thread 8092 * after ensuring that there are no user space tasks left on the CPU. 8093 * 8094 * If there is a lazy mm in use on the hotplug thread, drop it and 8095 * switch to init_mm. 8096 * 8097 * The reference count on init_mm is dropped in finish_cpu(). 8098 */ 8099 static void sched_force_init_mm(void) 8100 { 8101 struct mm_struct *mm = current->active_mm; 8102 8103 if (mm != &init_mm) { 8104 mmgrab_lazy_tlb(&init_mm); 8105 local_irq_disable(); 8106 current->active_mm = &init_mm; 8107 switch_mm_irqs_off(mm, &init_mm, current); 8108 local_irq_enable(); 8109 finish_arch_post_lock_switch(); 8110 mmdrop_lazy_tlb(mm); 8111 } 8112 8113 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 8114 } 8115 8116 static int __balance_push_cpu_stop(void *arg) 8117 { 8118 struct task_struct *p = arg; 8119 struct rq *rq = this_rq(); 8120 struct rq_flags rf; 8121 int cpu; 8122 8123 raw_spin_lock_irq(&p->pi_lock); 8124 rq_lock(rq, &rf); 8125 8126 update_rq_clock(rq); 8127 8128 if (task_rq(p) == rq && task_on_rq_queued(p)) { 8129 cpu = select_fallback_rq(rq->cpu, p); 8130 rq = __migrate_task(rq, &rf, p, cpu); 8131 } 8132 8133 rq_unlock(rq, &rf); 8134 raw_spin_unlock_irq(&p->pi_lock); 8135 8136 put_task_struct(p); 8137 8138 return 0; 8139 } 8140 8141 static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 8142 8143 /* 8144 * Ensure we only run per-cpu kthreads once the CPU goes !active. 8145 * 8146 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 8147 * effective when the hotplug motion is down. 8148 */ 8149 static void balance_push(struct rq *rq) 8150 { 8151 struct task_struct *push_task = rq->curr; 8152 8153 lockdep_assert_rq_held(rq); 8154 8155 /* 8156 * Ensure the thing is persistent until balance_push_set(.on = false); 8157 */ 8158 rq->balance_callback = &balance_push_callback; 8159 8160 /* 8161 * Only active while going offline and when invoked on the outgoing 8162 * CPU. 8163 */ 8164 if (!cpu_dying(rq->cpu) || rq != this_rq()) 8165 return; 8166 8167 /* 8168 * Both the cpu-hotplug and stop task are in this case and are 8169 * required to complete the hotplug process. 8170 */ 8171 if (kthread_is_per_cpu(push_task) || 8172 is_migration_disabled(push_task)) { 8173 8174 /* 8175 * If this is the idle task on the outgoing CPU try to wake 8176 * up the hotplug control thread which might wait for the 8177 * last task to vanish. The rcuwait_active() check is 8178 * accurate here because the waiter is pinned on this CPU 8179 * and can't obviously be running in parallel. 8180 * 8181 * On RT kernels this also has to check whether there are 8182 * pinned and scheduled out tasks on the runqueue. They 8183 * need to leave the migrate disabled section first. 8184 */ 8185 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 8186 rcuwait_active(&rq->hotplug_wait)) { 8187 raw_spin_rq_unlock(rq); 8188 rcuwait_wake_up(&rq->hotplug_wait); 8189 raw_spin_rq_lock(rq); 8190 } 8191 return; 8192 } 8193 8194 get_task_struct(push_task); 8195 /* 8196 * Temporarily drop rq->lock such that we can wake-up the stop task. 8197 * Both preemption and IRQs are still disabled. 8198 */ 8199 preempt_disable(); 8200 raw_spin_rq_unlock(rq); 8201 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 8202 this_cpu_ptr(&push_work)); 8203 preempt_enable(); 8204 /* 8205 * At this point need_resched() is true and we'll take the loop in 8206 * schedule(). The next pick is obviously going to be the stop task 8207 * which kthread_is_per_cpu() and will push this task away. 8208 */ 8209 raw_spin_rq_lock(rq); 8210 } 8211 8212 static void balance_push_set(int cpu, bool on) 8213 { 8214 struct rq *rq = cpu_rq(cpu); 8215 struct rq_flags rf; 8216 8217 rq_lock_irqsave(rq, &rf); 8218 if (on) { 8219 WARN_ON_ONCE(rq->balance_callback); 8220 rq->balance_callback = &balance_push_callback; 8221 } else if (rq->balance_callback == &balance_push_callback) { 8222 rq->balance_callback = NULL; 8223 } 8224 rq_unlock_irqrestore(rq, &rf); 8225 } 8226 8227 /* 8228 * Invoked from a CPUs hotplug control thread after the CPU has been marked 8229 * inactive. All tasks which are not per CPU kernel threads are either 8230 * pushed off this CPU now via balance_push() or placed on a different CPU 8231 * during wakeup. Wait until the CPU is quiescent. 8232 */ 8233 static void balance_hotplug_wait(void) 8234 { 8235 struct rq *rq = this_rq(); 8236 8237 rcuwait_wait_event(&rq->hotplug_wait, 8238 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 8239 TASK_UNINTERRUPTIBLE); 8240 } 8241 8242 #else /* !CONFIG_HOTPLUG_CPU: */ 8243 8244 static inline void balance_push(struct rq *rq) 8245 { 8246 } 8247 8248 static inline void balance_push_set(int cpu, bool on) 8249 { 8250 } 8251 8252 static inline void balance_hotplug_wait(void) 8253 { 8254 } 8255 8256 #endif /* !CONFIG_HOTPLUG_CPU */ 8257 8258 void set_rq_online(struct rq *rq) 8259 { 8260 if (!rq->online) { 8261 const struct sched_class *class; 8262 8263 cpumask_set_cpu(rq->cpu, rq->rd->online); 8264 rq->online = 1; 8265 8266 for_each_class(class) { 8267 if (class->rq_online) 8268 class->rq_online(rq); 8269 } 8270 } 8271 } 8272 8273 void set_rq_offline(struct rq *rq) 8274 { 8275 if (rq->online) { 8276 const struct sched_class *class; 8277 8278 update_rq_clock(rq); 8279 for_each_class(class) { 8280 if (class->rq_offline) 8281 class->rq_offline(rq); 8282 } 8283 8284 cpumask_clear_cpu(rq->cpu, rq->rd->online); 8285 rq->online = 0; 8286 } 8287 } 8288 8289 static inline void sched_set_rq_online(struct rq *rq, int cpu) 8290 { 8291 struct rq_flags rf; 8292 8293 rq_lock_irqsave(rq, &rf); 8294 if (rq->rd) { 8295 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8296 set_rq_online(rq); 8297 } 8298 rq_unlock_irqrestore(rq, &rf); 8299 } 8300 8301 static inline void sched_set_rq_offline(struct rq *rq, int cpu) 8302 { 8303 struct rq_flags rf; 8304 8305 rq_lock_irqsave(rq, &rf); 8306 if (rq->rd) { 8307 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8308 set_rq_offline(rq); 8309 } 8310 rq_unlock_irqrestore(rq, &rf); 8311 } 8312 8313 /* 8314 * used to mark begin/end of suspend/resume: 8315 */ 8316 static int num_cpus_frozen; 8317 8318 /* 8319 * Update cpusets according to cpu_active mask. If cpusets are 8320 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 8321 * around partition_sched_domains(). 8322 * 8323 * If we come here as part of a suspend/resume, don't touch cpusets because we 8324 * want to restore it back to its original state upon resume anyway. 8325 */ 8326 static void cpuset_cpu_active(void) 8327 { 8328 if (cpuhp_tasks_frozen) { 8329 /* 8330 * num_cpus_frozen tracks how many CPUs are involved in suspend 8331 * resume sequence. As long as this is not the last online 8332 * operation in the resume sequence, just build a single sched 8333 * domain, ignoring cpusets. 8334 */ 8335 cpuset_reset_sched_domains(); 8336 if (--num_cpus_frozen) 8337 return; 8338 /* 8339 * This is the last CPU online operation. So fall through and 8340 * restore the original sched domains by considering the 8341 * cpuset configurations. 8342 */ 8343 cpuset_force_rebuild(); 8344 } 8345 cpuset_update_active_cpus(); 8346 } 8347 8348 static void cpuset_cpu_inactive(unsigned int cpu) 8349 { 8350 if (!cpuhp_tasks_frozen) { 8351 cpuset_update_active_cpus(); 8352 } else { 8353 num_cpus_frozen++; 8354 cpuset_reset_sched_domains(); 8355 } 8356 } 8357 8358 static inline void sched_smt_present_inc(int cpu) 8359 { 8360 #ifdef CONFIG_SCHED_SMT 8361 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8362 static_branch_inc_cpuslocked(&sched_smt_present); 8363 #endif 8364 } 8365 8366 static inline void sched_smt_present_dec(int cpu) 8367 { 8368 #ifdef CONFIG_SCHED_SMT 8369 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8370 static_branch_dec_cpuslocked(&sched_smt_present); 8371 #endif 8372 } 8373 8374 int sched_cpu_activate(unsigned int cpu) 8375 { 8376 struct rq *rq = cpu_rq(cpu); 8377 8378 /* 8379 * Clear the balance_push callback and prepare to schedule 8380 * regular tasks. 8381 */ 8382 balance_push_set(cpu, false); 8383 8384 /* 8385 * When going up, increment the number of cores with SMT present. 8386 */ 8387 sched_smt_present_inc(cpu); 8388 set_cpu_active(cpu, true); 8389 8390 if (sched_smp_initialized) { 8391 sched_update_numa(cpu, true); 8392 sched_domains_numa_masks_set(cpu); 8393 cpuset_cpu_active(); 8394 } 8395 8396 scx_rq_activate(rq); 8397 8398 /* 8399 * Put the rq online, if not already. This happens: 8400 * 8401 * 1) In the early boot process, because we build the real domains 8402 * after all CPUs have been brought up. 8403 * 8404 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 8405 * domains. 8406 */ 8407 sched_set_rq_online(rq, cpu); 8408 8409 return 0; 8410 } 8411 8412 int sched_cpu_deactivate(unsigned int cpu) 8413 { 8414 struct rq *rq = cpu_rq(cpu); 8415 int ret; 8416 8417 ret = dl_bw_deactivate(cpu); 8418 8419 if (ret) 8420 return ret; 8421 8422 /* 8423 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 8424 * load balancing when not active 8425 */ 8426 nohz_balance_exit_idle(rq); 8427 8428 set_cpu_active(cpu, false); 8429 8430 /* 8431 * From this point forward, this CPU will refuse to run any task that 8432 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 8433 * push those tasks away until this gets cleared, see 8434 * sched_cpu_dying(). 8435 */ 8436 balance_push_set(cpu, true); 8437 8438 /* 8439 * We've cleared cpu_active_mask / set balance_push, wait for all 8440 * preempt-disabled and RCU users of this state to go away such that 8441 * all new such users will observe it. 8442 * 8443 * Specifically, we rely on ttwu to no longer target this CPU, see 8444 * ttwu_queue_cond() and is_cpu_allowed(). 8445 * 8446 * Do sync before park smpboot threads to take care the RCU boost case. 8447 */ 8448 synchronize_rcu(); 8449 8450 sched_set_rq_offline(rq, cpu); 8451 8452 scx_rq_deactivate(rq); 8453 8454 /* 8455 * When going down, decrement the number of cores with SMT present. 8456 */ 8457 sched_smt_present_dec(cpu); 8458 8459 #ifdef CONFIG_SCHED_SMT 8460 sched_core_cpu_deactivate(cpu); 8461 #endif 8462 8463 if (!sched_smp_initialized) 8464 return 0; 8465 8466 sched_update_numa(cpu, false); 8467 cpuset_cpu_inactive(cpu); 8468 sched_domains_numa_masks_clear(cpu); 8469 return 0; 8470 } 8471 8472 static void sched_rq_cpu_starting(unsigned int cpu) 8473 { 8474 struct rq *rq = cpu_rq(cpu); 8475 8476 rq->calc_load_update = calc_load_update; 8477 update_max_interval(); 8478 } 8479 8480 int sched_cpu_starting(unsigned int cpu) 8481 { 8482 sched_core_cpu_starting(cpu); 8483 sched_rq_cpu_starting(cpu); 8484 sched_tick_start(cpu); 8485 return 0; 8486 } 8487 8488 #ifdef CONFIG_HOTPLUG_CPU 8489 8490 /* 8491 * Invoked immediately before the stopper thread is invoked to bring the 8492 * CPU down completely. At this point all per CPU kthreads except the 8493 * hotplug thread (current) and the stopper thread (inactive) have been 8494 * either parked or have been unbound from the outgoing CPU. Ensure that 8495 * any of those which might be on the way out are gone. 8496 * 8497 * If after this point a bound task is being woken on this CPU then the 8498 * responsible hotplug callback has failed to do it's job. 8499 * sched_cpu_dying() will catch it with the appropriate fireworks. 8500 */ 8501 int sched_cpu_wait_empty(unsigned int cpu) 8502 { 8503 balance_hotplug_wait(); 8504 sched_force_init_mm(); 8505 return 0; 8506 } 8507 8508 /* 8509 * Since this CPU is going 'away' for a while, fold any nr_active delta we 8510 * might have. Called from the CPU stopper task after ensuring that the 8511 * stopper is the last running task on the CPU, so nr_active count is 8512 * stable. We need to take the tear-down thread which is calling this into 8513 * account, so we hand in adjust = 1 to the load calculation. 8514 * 8515 * Also see the comment "Global load-average calculations". 8516 */ 8517 static void calc_load_migrate(struct rq *rq) 8518 { 8519 long delta = calc_load_fold_active(rq, 1); 8520 8521 if (delta) 8522 atomic_long_add(delta, &calc_load_tasks); 8523 } 8524 8525 static void dump_rq_tasks(struct rq *rq, const char *loglvl) 8526 { 8527 struct task_struct *g, *p; 8528 int cpu = cpu_of(rq); 8529 8530 lockdep_assert_rq_held(rq); 8531 8532 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 8533 for_each_process_thread(g, p) { 8534 if (task_cpu(p) != cpu) 8535 continue; 8536 8537 if (!task_on_rq_queued(p)) 8538 continue; 8539 8540 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 8541 } 8542 } 8543 8544 int sched_cpu_dying(unsigned int cpu) 8545 { 8546 struct rq *rq = cpu_rq(cpu); 8547 struct rq_flags rf; 8548 8549 /* Handle pending wakeups and then migrate everything off */ 8550 sched_tick_stop(cpu); 8551 8552 rq_lock_irqsave(rq, &rf); 8553 update_rq_clock(rq); 8554 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 8555 WARN(true, "Dying CPU not properly vacated!"); 8556 dump_rq_tasks(rq, KERN_WARNING); 8557 } 8558 dl_server_stop(&rq->fair_server); 8559 rq_unlock_irqrestore(rq, &rf); 8560 8561 calc_load_migrate(rq); 8562 update_max_interval(); 8563 hrtick_clear(rq); 8564 sched_core_cpu_dying(cpu); 8565 return 0; 8566 } 8567 #endif /* CONFIG_HOTPLUG_CPU */ 8568 8569 void __init sched_init_smp(void) 8570 { 8571 sched_init_numa(NUMA_NO_NODE); 8572 8573 prandom_init_once(&sched_rnd_state); 8574 8575 /* 8576 * There's no userspace yet to cause hotplug operations; hence all the 8577 * CPU masks are stable and all blatant races in the below code cannot 8578 * happen. 8579 */ 8580 sched_domains_mutex_lock(); 8581 sched_init_domains(cpu_active_mask); 8582 sched_domains_mutex_unlock(); 8583 8584 /* Move init over to a non-isolated CPU */ 8585 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) 8586 BUG(); 8587 current->flags &= ~PF_NO_SETAFFINITY; 8588 sched_init_granularity(); 8589 8590 init_sched_rt_class(); 8591 init_sched_dl_class(); 8592 8593 sched_init_dl_servers(); 8594 8595 sched_smp_initialized = true; 8596 } 8597 8598 static int __init migration_init(void) 8599 { 8600 sched_cpu_starting(smp_processor_id()); 8601 return 0; 8602 } 8603 early_initcall(migration_init); 8604 8605 int in_sched_functions(unsigned long addr) 8606 { 8607 return in_lock_functions(addr) || 8608 (addr >= (unsigned long)__sched_text_start 8609 && addr < (unsigned long)__sched_text_end); 8610 } 8611 8612 #ifdef CONFIG_CGROUP_SCHED 8613 /* 8614 * Default task group. 8615 * Every task in system belongs to this group at bootup. 8616 */ 8617 struct task_group root_task_group; 8618 LIST_HEAD(task_groups); 8619 8620 /* Cacheline aligned slab cache for task_group */ 8621 static struct kmem_cache *task_group_cache __ro_after_init; 8622 #endif 8623 8624 void __init sched_init(void) 8625 { 8626 unsigned long ptr = 0; 8627 int i; 8628 8629 /* Make sure the linker didn't screw up */ 8630 BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); 8631 BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); 8632 BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); 8633 BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); 8634 #ifdef CONFIG_SCHED_CLASS_EXT 8635 BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); 8636 BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); 8637 #endif 8638 8639 wait_bit_init(); 8640 8641 #ifdef CONFIG_FAIR_GROUP_SCHED 8642 ptr += 2 * nr_cpu_ids * sizeof(void **); 8643 #endif 8644 #ifdef CONFIG_RT_GROUP_SCHED 8645 ptr += 2 * nr_cpu_ids * sizeof(void **); 8646 #endif 8647 if (ptr) { 8648 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 8649 8650 #ifdef CONFIG_FAIR_GROUP_SCHED 8651 root_task_group.se = (struct sched_entity **)ptr; 8652 ptr += nr_cpu_ids * sizeof(void **); 8653 8654 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 8655 ptr += nr_cpu_ids * sizeof(void **); 8656 8657 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 8658 init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); 8659 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8660 #ifdef CONFIG_EXT_GROUP_SCHED 8661 scx_tg_init(&root_task_group); 8662 #endif /* CONFIG_EXT_GROUP_SCHED */ 8663 #ifdef CONFIG_RT_GROUP_SCHED 8664 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 8665 ptr += nr_cpu_ids * sizeof(void **); 8666 8667 root_task_group.rt_rq = (struct rt_rq **)ptr; 8668 ptr += nr_cpu_ids * sizeof(void **); 8669 8670 #endif /* CONFIG_RT_GROUP_SCHED */ 8671 } 8672 8673 init_defrootdomain(); 8674 8675 #ifdef CONFIG_RT_GROUP_SCHED 8676 init_rt_bandwidth(&root_task_group.rt_bandwidth, 8677 global_rt_period(), global_rt_runtime()); 8678 #endif /* CONFIG_RT_GROUP_SCHED */ 8679 8680 #ifdef CONFIG_CGROUP_SCHED 8681 task_group_cache = KMEM_CACHE(task_group, 0); 8682 8683 list_add(&root_task_group.list, &task_groups); 8684 INIT_LIST_HEAD(&root_task_group.children); 8685 INIT_LIST_HEAD(&root_task_group.siblings); 8686 autogroup_init(&init_task); 8687 #endif /* CONFIG_CGROUP_SCHED */ 8688 8689 for_each_possible_cpu(i) { 8690 struct rq *rq; 8691 8692 rq = cpu_rq(i); 8693 raw_spin_lock_init(&rq->__lock); 8694 rq->nr_running = 0; 8695 rq->calc_load_active = 0; 8696 rq->calc_load_update = jiffies + LOAD_FREQ; 8697 init_cfs_rq(&rq->cfs); 8698 init_rt_rq(&rq->rt); 8699 init_dl_rq(&rq->dl); 8700 #ifdef CONFIG_FAIR_GROUP_SCHED 8701 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 8702 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 8703 /* 8704 * How much CPU bandwidth does root_task_group get? 8705 * 8706 * In case of task-groups formed through the cgroup filesystem, it 8707 * gets 100% of the CPU resources in the system. This overall 8708 * system CPU resource is divided among the tasks of 8709 * root_task_group and its child task-groups in a fair manner, 8710 * based on each entity's (task or task-group's) weight 8711 * (se->load.weight). 8712 * 8713 * In other words, if root_task_group has 10 tasks of weight 8714 * 1024) and two child groups A0 and A1 (of weight 1024 each), 8715 * then A0's share of the CPU resource is: 8716 * 8717 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 8718 * 8719 * We achieve this by letting root_task_group's tasks sit 8720 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 8721 */ 8722 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 8723 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8724 8725 #ifdef CONFIG_RT_GROUP_SCHED 8726 /* 8727 * This is required for init cpu because rt.c:__enable_runtime() 8728 * starts working after scheduler_running, which is not the case 8729 * yet. 8730 */ 8731 rq->rt.rt_runtime = global_rt_runtime(); 8732 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 8733 #endif 8734 rq->sd = NULL; 8735 rq->rd = NULL; 8736 rq->cpu_capacity = SCHED_CAPACITY_SCALE; 8737 rq->balance_callback = &balance_push_callback; 8738 rq->active_balance = 0; 8739 rq->next_balance = jiffies; 8740 rq->push_cpu = 0; 8741 rq->cpu = i; 8742 rq->online = 0; 8743 rq->idle_stamp = 0; 8744 rq->avg_idle = 2*sysctl_sched_migration_cost; 8745 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 8746 8747 INIT_LIST_HEAD(&rq->cfs_tasks); 8748 8749 rq_attach_root(rq, &def_root_domain); 8750 #ifdef CONFIG_NO_HZ_COMMON 8751 rq->last_blocked_load_update_tick = jiffies; 8752 atomic_set(&rq->nohz_flags, 0); 8753 8754 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 8755 #endif 8756 #ifdef CONFIG_HOTPLUG_CPU 8757 rcuwait_init(&rq->hotplug_wait); 8758 #endif 8759 hrtick_rq_init(rq); 8760 atomic_set(&rq->nr_iowait, 0); 8761 fair_server_init(rq); 8762 8763 #ifdef CONFIG_SCHED_CORE 8764 rq->core = rq; 8765 rq->core_pick = NULL; 8766 rq->core_dl_server = NULL; 8767 rq->core_enabled = 0; 8768 rq->core_tree = RB_ROOT; 8769 rq->core_forceidle_count = 0; 8770 rq->core_forceidle_occupation = 0; 8771 rq->core_forceidle_start = 0; 8772 8773 rq->core_cookie = 0UL; 8774 #endif 8775 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); 8776 } 8777 8778 set_load_weight(&init_task, false); 8779 init_task.se.slice = sysctl_sched_base_slice, 8780 8781 /* 8782 * The boot idle thread does lazy MMU switching as well: 8783 */ 8784 mmgrab_lazy_tlb(&init_mm); 8785 enter_lazy_tlb(&init_mm, current); 8786 8787 /* 8788 * The idle task doesn't need the kthread struct to function, but it 8789 * is dressed up as a per-CPU kthread and thus needs to play the part 8790 * if we want to avoid special-casing it in code that deals with per-CPU 8791 * kthreads. 8792 */ 8793 WARN_ON(!set_kthread_struct(current)); 8794 8795 /* 8796 * Make us the idle thread. Technically, schedule() should not be 8797 * called from this thread, however somewhere below it might be, 8798 * but because we are the idle thread, we just pick up running again 8799 * when this runqueue becomes "idle". 8800 */ 8801 __sched_fork(0, current); 8802 init_idle(current, smp_processor_id()); 8803 8804 calc_load_update = jiffies + LOAD_FREQ; 8805 8806 idle_thread_set_boot_cpu(); 8807 8808 balance_push_set(smp_processor_id(), false); 8809 init_sched_fair_class(); 8810 init_sched_ext_class(); 8811 8812 psi_init(); 8813 8814 init_uclamp(); 8815 8816 preempt_dynamic_init(); 8817 8818 scheduler_running = 1; 8819 } 8820 8821 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 8822 8823 void __might_sleep(const char *file, int line) 8824 { 8825 unsigned int state = get_current_state(); 8826 /* 8827 * Blocking primitives will set (and therefore destroy) current->state, 8828 * since we will exit with TASK_RUNNING make sure we enter with it, 8829 * otherwise we will destroy state. 8830 */ 8831 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 8832 "do not call blocking ops when !TASK_RUNNING; " 8833 "state=%x set at [<%p>] %pS\n", state, 8834 (void *)current->task_state_change, 8835 (void *)current->task_state_change); 8836 8837 __might_resched(file, line, 0); 8838 } 8839 EXPORT_SYMBOL(__might_sleep); 8840 8841 static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) 8842 { 8843 if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) 8844 return; 8845 8846 if (preempt_count() == preempt_offset) 8847 return; 8848 8849 pr_err("Preemption disabled at:"); 8850 print_ip_sym(KERN_ERR, ip); 8851 } 8852 8853 static inline bool resched_offsets_ok(unsigned int offsets) 8854 { 8855 unsigned int nested = preempt_count(); 8856 8857 nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; 8858 8859 return nested == offsets; 8860 } 8861 8862 void __might_resched(const char *file, int line, unsigned int offsets) 8863 { 8864 /* Ratelimiting timestamp: */ 8865 static unsigned long prev_jiffy; 8866 8867 unsigned long preempt_disable_ip; 8868 8869 /* WARN_ON_ONCE() by default, no rate limit required: */ 8870 rcu_sleep_check(); 8871 8872 if ((resched_offsets_ok(offsets) && !irqs_disabled() && 8873 !is_idle_task(current) && !current->non_block_count) || 8874 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 8875 oops_in_progress) 8876 return; 8877 8878 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8879 return; 8880 prev_jiffy = jiffies; 8881 8882 /* Save this before calling printk(), since that will clobber it: */ 8883 preempt_disable_ip = get_preempt_disable_ip(current); 8884 8885 pr_err("BUG: sleeping function called from invalid context at %s:%d\n", 8886 file, line); 8887 pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 8888 in_atomic(), irqs_disabled(), current->non_block_count, 8889 current->pid, current->comm); 8890 pr_err("preempt_count: %x, expected: %x\n", preempt_count(), 8891 offsets & MIGHT_RESCHED_PREEMPT_MASK); 8892 8893 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { 8894 pr_err("RCU nest depth: %d, expected: %u\n", 8895 rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); 8896 } 8897 8898 if (task_stack_end_corrupted(current)) 8899 pr_emerg("Thread overran stack, or stack corrupted\n"); 8900 8901 debug_show_held_locks(current); 8902 if (irqs_disabled()) 8903 print_irqtrace_events(current); 8904 8905 print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, 8906 preempt_disable_ip); 8907 8908 dump_stack(); 8909 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8910 } 8911 EXPORT_SYMBOL(__might_resched); 8912 8913 void __cant_sleep(const char *file, int line, int preempt_offset) 8914 { 8915 static unsigned long prev_jiffy; 8916 8917 if (irqs_disabled()) 8918 return; 8919 8920 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8921 return; 8922 8923 if (preempt_count() > preempt_offset) 8924 return; 8925 8926 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8927 return; 8928 prev_jiffy = jiffies; 8929 8930 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 8931 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 8932 in_atomic(), irqs_disabled(), 8933 current->pid, current->comm); 8934 8935 debug_show_held_locks(current); 8936 dump_stack(); 8937 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8938 } 8939 EXPORT_SYMBOL_GPL(__cant_sleep); 8940 8941 # ifdef CONFIG_SMP 8942 void __cant_migrate(const char *file, int line) 8943 { 8944 static unsigned long prev_jiffy; 8945 8946 if (irqs_disabled()) 8947 return; 8948 8949 if (is_migration_disabled(current)) 8950 return; 8951 8952 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8953 return; 8954 8955 if (preempt_count() > 0) 8956 return; 8957 8958 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8959 return; 8960 prev_jiffy = jiffies; 8961 8962 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 8963 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 8964 in_atomic(), irqs_disabled(), is_migration_disabled(current), 8965 current->pid, current->comm); 8966 8967 debug_show_held_locks(current); 8968 dump_stack(); 8969 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8970 } 8971 EXPORT_SYMBOL_GPL(__cant_migrate); 8972 # endif /* CONFIG_SMP */ 8973 #endif /* CONFIG_DEBUG_ATOMIC_SLEEP */ 8974 8975 #ifdef CONFIG_MAGIC_SYSRQ 8976 void normalize_rt_tasks(void) 8977 { 8978 struct task_struct *g, *p; 8979 struct sched_attr attr = { 8980 .sched_policy = SCHED_NORMAL, 8981 }; 8982 8983 read_lock(&tasklist_lock); 8984 for_each_process_thread(g, p) { 8985 /* 8986 * Only normalize user tasks: 8987 */ 8988 if (p->flags & PF_KTHREAD) 8989 continue; 8990 8991 p->se.exec_start = 0; 8992 schedstat_set(p->stats.wait_start, 0); 8993 schedstat_set(p->stats.sleep_start, 0); 8994 schedstat_set(p->stats.block_start, 0); 8995 8996 if (!rt_or_dl_task(p)) { 8997 /* 8998 * Renice negative nice level userspace 8999 * tasks back to 0: 9000 */ 9001 if (task_nice(p) < 0) 9002 set_user_nice(p, 0); 9003 continue; 9004 } 9005 9006 __sched_setscheduler(p, &attr, false, false); 9007 } 9008 read_unlock(&tasklist_lock); 9009 } 9010 9011 #endif /* CONFIG_MAGIC_SYSRQ */ 9012 9013 #ifdef CONFIG_KGDB_KDB 9014 /* 9015 * These functions are only useful for KDB. 9016 * 9017 * They can only be called when the whole system has been 9018 * stopped - every CPU needs to be quiescent, and no scheduling 9019 * activity can take place. Using them for anything else would 9020 * be a serious bug, and as a result, they aren't even visible 9021 * under any other configuration. 9022 */ 9023 9024 /** 9025 * curr_task - return the current task for a given CPU. 9026 * @cpu: the processor in question. 9027 * 9028 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 9029 * 9030 * Return: The current task for @cpu. 9031 */ 9032 struct task_struct *curr_task(int cpu) 9033 { 9034 return cpu_curr(cpu); 9035 } 9036 9037 #endif /* CONFIG_KGDB_KDB */ 9038 9039 #ifdef CONFIG_CGROUP_SCHED 9040 /* task_group_lock serializes the addition/removal of task groups */ 9041 static DEFINE_SPINLOCK(task_group_lock); 9042 9043 static inline void alloc_uclamp_sched_group(struct task_group *tg, 9044 struct task_group *parent) 9045 { 9046 #ifdef CONFIG_UCLAMP_TASK_GROUP 9047 enum uclamp_id clamp_id; 9048 9049 for_each_clamp_id(clamp_id) { 9050 uclamp_se_set(&tg->uclamp_req[clamp_id], 9051 uclamp_none(clamp_id), false); 9052 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 9053 } 9054 #endif 9055 } 9056 9057 static void sched_free_group(struct task_group *tg) 9058 { 9059 free_fair_sched_group(tg); 9060 free_rt_sched_group(tg); 9061 autogroup_free(tg); 9062 kmem_cache_free(task_group_cache, tg); 9063 } 9064 9065 static void sched_free_group_rcu(struct rcu_head *rcu) 9066 { 9067 sched_free_group(container_of(rcu, struct task_group, rcu)); 9068 } 9069 9070 static void sched_unregister_group(struct task_group *tg) 9071 { 9072 unregister_fair_sched_group(tg); 9073 unregister_rt_sched_group(tg); 9074 /* 9075 * We have to wait for yet another RCU grace period to expire, as 9076 * print_cfs_stats() might run concurrently. 9077 */ 9078 call_rcu(&tg->rcu, sched_free_group_rcu); 9079 } 9080 9081 /* allocate runqueue etc for a new task group */ 9082 struct task_group *sched_create_group(struct task_group *parent) 9083 { 9084 struct task_group *tg; 9085 9086 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 9087 if (!tg) 9088 return ERR_PTR(-ENOMEM); 9089 9090 if (!alloc_fair_sched_group(tg, parent)) 9091 goto err; 9092 9093 if (!alloc_rt_sched_group(tg, parent)) 9094 goto err; 9095 9096 scx_tg_init(tg); 9097 alloc_uclamp_sched_group(tg, parent); 9098 9099 return tg; 9100 9101 err: 9102 sched_free_group(tg); 9103 return ERR_PTR(-ENOMEM); 9104 } 9105 9106 void sched_online_group(struct task_group *tg, struct task_group *parent) 9107 { 9108 unsigned long flags; 9109 9110 spin_lock_irqsave(&task_group_lock, flags); 9111 list_add_tail_rcu(&tg->list, &task_groups); 9112 9113 /* Root should already exist: */ 9114 WARN_ON(!parent); 9115 9116 tg->parent = parent; 9117 INIT_LIST_HEAD(&tg->children); 9118 list_add_rcu(&tg->siblings, &parent->children); 9119 spin_unlock_irqrestore(&task_group_lock, flags); 9120 9121 online_fair_sched_group(tg); 9122 } 9123 9124 /* RCU callback to free various structures associated with a task group */ 9125 static void sched_unregister_group_rcu(struct rcu_head *rhp) 9126 { 9127 /* Now it should be safe to free those cfs_rqs: */ 9128 sched_unregister_group(container_of(rhp, struct task_group, rcu)); 9129 } 9130 9131 void sched_destroy_group(struct task_group *tg) 9132 { 9133 /* Wait for possible concurrent references to cfs_rqs complete: */ 9134 call_rcu(&tg->rcu, sched_unregister_group_rcu); 9135 } 9136 9137 void sched_release_group(struct task_group *tg) 9138 { 9139 unsigned long flags; 9140 9141 /* 9142 * Unlink first, to avoid walk_tg_tree_from() from finding us (via 9143 * sched_cfs_period_timer()). 9144 * 9145 * For this to be effective, we have to wait for all pending users of 9146 * this task group to leave their RCU critical section to ensure no new 9147 * user will see our dying task group any more. Specifically ensure 9148 * that tg_unthrottle_up() won't add decayed cfs_rq's to it. 9149 * 9150 * We therefore defer calling unregister_fair_sched_group() to 9151 * sched_unregister_group() which is guarantied to get called only after the 9152 * current RCU grace period has expired. 9153 */ 9154 spin_lock_irqsave(&task_group_lock, flags); 9155 list_del_rcu(&tg->list); 9156 list_del_rcu(&tg->siblings); 9157 spin_unlock_irqrestore(&task_group_lock, flags); 9158 } 9159 9160 static void sched_change_group(struct task_struct *tsk) 9161 { 9162 struct task_group *tg; 9163 9164 /* 9165 * All callers are synchronized by task_rq_lock(); we do not use RCU 9166 * which is pointless here. Thus, we pass "true" to task_css_check() 9167 * to prevent lockdep warnings. 9168 */ 9169 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 9170 struct task_group, css); 9171 tg = autogroup_task_group(tsk, tg); 9172 tsk->sched_task_group = tg; 9173 9174 #ifdef CONFIG_FAIR_GROUP_SCHED 9175 if (tsk->sched_class->task_change_group) 9176 tsk->sched_class->task_change_group(tsk); 9177 else 9178 #endif 9179 set_task_rq(tsk, task_cpu(tsk)); 9180 } 9181 9182 /* 9183 * Change task's runqueue when it moves between groups. 9184 * 9185 * The caller of this function should have put the task in its new group by 9186 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 9187 * its new group. 9188 */ 9189 void sched_move_task(struct task_struct *tsk, bool for_autogroup) 9190 { 9191 unsigned int queue_flags = 9192 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 9193 bool resched = false; 9194 struct rq *rq; 9195 9196 CLASS(task_rq_lock, rq_guard)(tsk); 9197 rq = rq_guard.rq; 9198 9199 update_rq_clock(rq); 9200 9201 scoped_guard (sched_change, tsk, queue_flags) { 9202 sched_change_group(tsk); 9203 if (!for_autogroup) 9204 scx_cgroup_move_task(tsk); 9205 if (scope->running) 9206 resched = true; 9207 } 9208 9209 if (resched) 9210 resched_curr(rq); 9211 } 9212 9213 static struct cgroup_subsys_state * 9214 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 9215 { 9216 struct task_group *parent = css_tg(parent_css); 9217 struct task_group *tg; 9218 9219 if (!parent) { 9220 /* This is early initialization for the top cgroup */ 9221 return &root_task_group.css; 9222 } 9223 9224 tg = sched_create_group(parent); 9225 if (IS_ERR(tg)) 9226 return ERR_PTR(-ENOMEM); 9227 9228 return &tg->css; 9229 } 9230 9231 /* Expose task group only after completing cgroup initialization */ 9232 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 9233 { 9234 struct task_group *tg = css_tg(css); 9235 struct task_group *parent = css_tg(css->parent); 9236 int ret; 9237 9238 ret = scx_tg_online(tg); 9239 if (ret) 9240 return ret; 9241 9242 if (parent) 9243 sched_online_group(tg, parent); 9244 9245 #ifdef CONFIG_UCLAMP_TASK_GROUP 9246 /* Propagate the effective uclamp value for the new group */ 9247 guard(mutex)(&uclamp_mutex); 9248 guard(rcu)(); 9249 cpu_util_update_eff(css); 9250 #endif 9251 9252 return 0; 9253 } 9254 9255 static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css) 9256 { 9257 struct task_group *tg = css_tg(css); 9258 9259 scx_tg_offline(tg); 9260 } 9261 9262 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 9263 { 9264 struct task_group *tg = css_tg(css); 9265 9266 sched_release_group(tg); 9267 } 9268 9269 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 9270 { 9271 struct task_group *tg = css_tg(css); 9272 9273 /* 9274 * Relies on the RCU grace period between css_released() and this. 9275 */ 9276 sched_unregister_group(tg); 9277 } 9278 9279 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 9280 { 9281 #ifdef CONFIG_RT_GROUP_SCHED 9282 struct task_struct *task; 9283 struct cgroup_subsys_state *css; 9284 9285 if (!rt_group_sched_enabled()) 9286 goto scx_check; 9287 9288 cgroup_taskset_for_each(task, css, tset) { 9289 if (!sched_rt_can_attach(css_tg(css), task)) 9290 return -EINVAL; 9291 } 9292 scx_check: 9293 #endif /* CONFIG_RT_GROUP_SCHED */ 9294 return scx_cgroup_can_attach(tset); 9295 } 9296 9297 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 9298 { 9299 struct task_struct *task; 9300 struct cgroup_subsys_state *css; 9301 9302 cgroup_taskset_for_each(task, css, tset) 9303 sched_move_task(task, false); 9304 } 9305 9306 static void cpu_cgroup_cancel_attach(struct cgroup_taskset *tset) 9307 { 9308 scx_cgroup_cancel_attach(tset); 9309 } 9310 9311 #ifdef CONFIG_UCLAMP_TASK_GROUP 9312 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 9313 { 9314 struct cgroup_subsys_state *top_css = css; 9315 struct uclamp_se *uc_parent = NULL; 9316 struct uclamp_se *uc_se = NULL; 9317 unsigned int eff[UCLAMP_CNT]; 9318 enum uclamp_id clamp_id; 9319 unsigned int clamps; 9320 9321 lockdep_assert_held(&uclamp_mutex); 9322 WARN_ON_ONCE(!rcu_read_lock_held()); 9323 9324 css_for_each_descendant_pre(css, top_css) { 9325 uc_parent = css_tg(css)->parent 9326 ? css_tg(css)->parent->uclamp : NULL; 9327 9328 for_each_clamp_id(clamp_id) { 9329 /* Assume effective clamps matches requested clamps */ 9330 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 9331 /* Cap effective clamps with parent's effective clamps */ 9332 if (uc_parent && 9333 eff[clamp_id] > uc_parent[clamp_id].value) { 9334 eff[clamp_id] = uc_parent[clamp_id].value; 9335 } 9336 } 9337 /* Ensure protection is always capped by limit */ 9338 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 9339 9340 /* Propagate most restrictive effective clamps */ 9341 clamps = 0x0; 9342 uc_se = css_tg(css)->uclamp; 9343 for_each_clamp_id(clamp_id) { 9344 if (eff[clamp_id] == uc_se[clamp_id].value) 9345 continue; 9346 uc_se[clamp_id].value = eff[clamp_id]; 9347 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 9348 clamps |= (0x1 << clamp_id); 9349 } 9350 if (!clamps) { 9351 css = css_rightmost_descendant(css); 9352 continue; 9353 } 9354 9355 /* Immediately update descendants RUNNABLE tasks */ 9356 uclamp_update_active_tasks(css); 9357 } 9358 } 9359 9360 /* 9361 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 9362 * C expression. Since there is no way to convert a macro argument (N) into a 9363 * character constant, use two levels of macros. 9364 */ 9365 #define _POW10(exp) ((unsigned int)1e##exp) 9366 #define POW10(exp) _POW10(exp) 9367 9368 struct uclamp_request { 9369 #define UCLAMP_PERCENT_SHIFT 2 9370 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 9371 s64 percent; 9372 u64 util; 9373 int ret; 9374 }; 9375 9376 static inline struct uclamp_request 9377 capacity_from_percent(char *buf) 9378 { 9379 struct uclamp_request req = { 9380 .percent = UCLAMP_PERCENT_SCALE, 9381 .util = SCHED_CAPACITY_SCALE, 9382 .ret = 0, 9383 }; 9384 9385 buf = strim(buf); 9386 if (strcmp(buf, "max")) { 9387 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 9388 &req.percent); 9389 if (req.ret) 9390 return req; 9391 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 9392 req.ret = -ERANGE; 9393 return req; 9394 } 9395 9396 req.util = req.percent << SCHED_CAPACITY_SHIFT; 9397 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 9398 } 9399 9400 return req; 9401 } 9402 9403 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 9404 size_t nbytes, loff_t off, 9405 enum uclamp_id clamp_id) 9406 { 9407 struct uclamp_request req; 9408 struct task_group *tg; 9409 9410 req = capacity_from_percent(buf); 9411 if (req.ret) 9412 return req.ret; 9413 9414 sched_uclamp_enable(); 9415 9416 guard(mutex)(&uclamp_mutex); 9417 guard(rcu)(); 9418 9419 tg = css_tg(of_css(of)); 9420 if (tg->uclamp_req[clamp_id].value != req.util) 9421 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 9422 9423 /* 9424 * Because of not recoverable conversion rounding we keep track of the 9425 * exact requested value 9426 */ 9427 tg->uclamp_pct[clamp_id] = req.percent; 9428 9429 /* Update effective clamps to track the most restrictive value */ 9430 cpu_util_update_eff(of_css(of)); 9431 9432 return nbytes; 9433 } 9434 9435 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 9436 char *buf, size_t nbytes, 9437 loff_t off) 9438 { 9439 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 9440 } 9441 9442 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 9443 char *buf, size_t nbytes, 9444 loff_t off) 9445 { 9446 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 9447 } 9448 9449 static inline void cpu_uclamp_print(struct seq_file *sf, 9450 enum uclamp_id clamp_id) 9451 { 9452 struct task_group *tg; 9453 u64 util_clamp; 9454 u64 percent; 9455 u32 rem; 9456 9457 scoped_guard (rcu) { 9458 tg = css_tg(seq_css(sf)); 9459 util_clamp = tg->uclamp_req[clamp_id].value; 9460 } 9461 9462 if (util_clamp == SCHED_CAPACITY_SCALE) { 9463 seq_puts(sf, "max\n"); 9464 return; 9465 } 9466 9467 percent = tg->uclamp_pct[clamp_id]; 9468 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 9469 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 9470 } 9471 9472 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 9473 { 9474 cpu_uclamp_print(sf, UCLAMP_MIN); 9475 return 0; 9476 } 9477 9478 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 9479 { 9480 cpu_uclamp_print(sf, UCLAMP_MAX); 9481 return 0; 9482 } 9483 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 9484 9485 #ifdef CONFIG_GROUP_SCHED_WEIGHT 9486 static unsigned long tg_weight(struct task_group *tg) 9487 { 9488 #ifdef CONFIG_FAIR_GROUP_SCHED 9489 return scale_load_down(tg->shares); 9490 #else 9491 return sched_weight_from_cgroup(tg->scx.weight); 9492 #endif 9493 } 9494 9495 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 9496 struct cftype *cftype, u64 shareval) 9497 { 9498 int ret; 9499 9500 if (shareval > scale_load_down(ULONG_MAX)) 9501 shareval = MAX_SHARES; 9502 ret = sched_group_set_shares(css_tg(css), scale_load(shareval)); 9503 if (!ret) 9504 scx_group_set_weight(css_tg(css), 9505 sched_weight_to_cgroup(shareval)); 9506 return ret; 9507 } 9508 9509 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 9510 struct cftype *cft) 9511 { 9512 return tg_weight(css_tg(css)); 9513 } 9514 #endif /* CONFIG_GROUP_SCHED_WEIGHT */ 9515 9516 #ifdef CONFIG_CFS_BANDWIDTH 9517 static DEFINE_MUTEX(cfs_constraints_mutex); 9518 9519 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 9520 9521 static int tg_set_cfs_bandwidth(struct task_group *tg, 9522 u64 period_us, u64 quota_us, u64 burst_us) 9523 { 9524 int i, ret = 0, runtime_enabled, runtime_was_enabled; 9525 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9526 u64 period, quota, burst; 9527 9528 period = (u64)period_us * NSEC_PER_USEC; 9529 9530 if (quota_us == RUNTIME_INF) 9531 quota = RUNTIME_INF; 9532 else 9533 quota = (u64)quota_us * NSEC_PER_USEC; 9534 9535 burst = (u64)burst_us * NSEC_PER_USEC; 9536 9537 /* 9538 * Prevent race between setting of cfs_rq->runtime_enabled and 9539 * unthrottle_offline_cfs_rqs(). 9540 */ 9541 guard(cpus_read_lock)(); 9542 guard(mutex)(&cfs_constraints_mutex); 9543 9544 ret = __cfs_schedulable(tg, period, quota); 9545 if (ret) 9546 return ret; 9547 9548 runtime_enabled = quota != RUNTIME_INF; 9549 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 9550 /* 9551 * If we need to toggle cfs_bandwidth_used, off->on must occur 9552 * before making related changes, and on->off must occur afterwards 9553 */ 9554 if (runtime_enabled && !runtime_was_enabled) 9555 cfs_bandwidth_usage_inc(); 9556 9557 scoped_guard (raw_spinlock_irq, &cfs_b->lock) { 9558 cfs_b->period = ns_to_ktime(period); 9559 cfs_b->quota = quota; 9560 cfs_b->burst = burst; 9561 9562 __refill_cfs_bandwidth_runtime(cfs_b); 9563 9564 /* 9565 * Restart the period timer (if active) to handle new 9566 * period expiry: 9567 */ 9568 if (runtime_enabled) 9569 start_cfs_bandwidth(cfs_b); 9570 } 9571 9572 for_each_online_cpu(i) { 9573 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 9574 struct rq *rq = cfs_rq->rq; 9575 9576 guard(rq_lock_irq)(rq); 9577 cfs_rq->runtime_enabled = runtime_enabled; 9578 cfs_rq->runtime_remaining = 1; 9579 9580 if (cfs_rq->throttled) 9581 unthrottle_cfs_rq(cfs_rq); 9582 } 9583 9584 if (runtime_was_enabled && !runtime_enabled) 9585 cfs_bandwidth_usage_dec(); 9586 9587 return 0; 9588 } 9589 9590 static u64 tg_get_cfs_period(struct task_group *tg) 9591 { 9592 u64 cfs_period_us; 9593 9594 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 9595 do_div(cfs_period_us, NSEC_PER_USEC); 9596 9597 return cfs_period_us; 9598 } 9599 9600 static u64 tg_get_cfs_quota(struct task_group *tg) 9601 { 9602 u64 quota_us; 9603 9604 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 9605 return RUNTIME_INF; 9606 9607 quota_us = tg->cfs_bandwidth.quota; 9608 do_div(quota_us, NSEC_PER_USEC); 9609 9610 return quota_us; 9611 } 9612 9613 static u64 tg_get_cfs_burst(struct task_group *tg) 9614 { 9615 u64 burst_us; 9616 9617 burst_us = tg->cfs_bandwidth.burst; 9618 do_div(burst_us, NSEC_PER_USEC); 9619 9620 return burst_us; 9621 } 9622 9623 struct cfs_schedulable_data { 9624 struct task_group *tg; 9625 u64 period, quota; 9626 }; 9627 9628 /* 9629 * normalize group quota/period to be quota/max_period 9630 * note: units are usecs 9631 */ 9632 static u64 normalize_cfs_quota(struct task_group *tg, 9633 struct cfs_schedulable_data *d) 9634 { 9635 u64 quota, period; 9636 9637 if (tg == d->tg) { 9638 period = d->period; 9639 quota = d->quota; 9640 } else { 9641 period = tg_get_cfs_period(tg); 9642 quota = tg_get_cfs_quota(tg); 9643 } 9644 9645 /* note: these should typically be equivalent */ 9646 if (quota == RUNTIME_INF || quota == -1) 9647 return RUNTIME_INF; 9648 9649 return to_ratio(period, quota); 9650 } 9651 9652 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 9653 { 9654 struct cfs_schedulable_data *d = data; 9655 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9656 s64 quota = 0, parent_quota = -1; 9657 9658 if (!tg->parent) { 9659 quota = RUNTIME_INF; 9660 } else { 9661 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 9662 9663 quota = normalize_cfs_quota(tg, d); 9664 parent_quota = parent_b->hierarchical_quota; 9665 9666 /* 9667 * Ensure max(child_quota) <= parent_quota. On cgroup2, 9668 * always take the non-RUNTIME_INF min. On cgroup1, only 9669 * inherit when no limit is set. In both cases this is used 9670 * by the scheduler to determine if a given CFS task has a 9671 * bandwidth constraint at some higher level. 9672 */ 9673 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 9674 if (quota == RUNTIME_INF) 9675 quota = parent_quota; 9676 else if (parent_quota != RUNTIME_INF) 9677 quota = min(quota, parent_quota); 9678 } else { 9679 if (quota == RUNTIME_INF) 9680 quota = parent_quota; 9681 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 9682 return -EINVAL; 9683 } 9684 } 9685 cfs_b->hierarchical_quota = quota; 9686 9687 return 0; 9688 } 9689 9690 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 9691 { 9692 struct cfs_schedulable_data data = { 9693 .tg = tg, 9694 .period = period, 9695 .quota = quota, 9696 }; 9697 9698 if (quota != RUNTIME_INF) { 9699 do_div(data.period, NSEC_PER_USEC); 9700 do_div(data.quota, NSEC_PER_USEC); 9701 } 9702 9703 guard(rcu)(); 9704 return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 9705 } 9706 9707 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 9708 { 9709 struct task_group *tg = css_tg(seq_css(sf)); 9710 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9711 9712 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 9713 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 9714 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 9715 9716 if (schedstat_enabled() && tg != &root_task_group) { 9717 struct sched_statistics *stats; 9718 u64 ws = 0; 9719 int i; 9720 9721 for_each_possible_cpu(i) { 9722 stats = __schedstats_from_se(tg->se[i]); 9723 ws += schedstat_val(stats->wait_sum); 9724 } 9725 9726 seq_printf(sf, "wait_sum %llu\n", ws); 9727 } 9728 9729 seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); 9730 seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); 9731 9732 return 0; 9733 } 9734 9735 static u64 throttled_time_self(struct task_group *tg) 9736 { 9737 int i; 9738 u64 total = 0; 9739 9740 for_each_possible_cpu(i) { 9741 total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); 9742 } 9743 9744 return total; 9745 } 9746 9747 static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v) 9748 { 9749 struct task_group *tg = css_tg(seq_css(sf)); 9750 9751 seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg)); 9752 9753 return 0; 9754 } 9755 #endif /* CONFIG_CFS_BANDWIDTH */ 9756 9757 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH 9758 const u64 max_bw_quota_period_us = 1 * USEC_PER_SEC; /* 1s */ 9759 static const u64 min_bw_quota_period_us = 1 * USEC_PER_MSEC; /* 1ms */ 9760 /* More than 203 days if BW_SHIFT equals 20. */ 9761 static const u64 max_bw_runtime_us = MAX_BW; 9762 9763 static void tg_bandwidth(struct task_group *tg, 9764 u64 *period_us_p, u64 *quota_us_p, u64 *burst_us_p) 9765 { 9766 #ifdef CONFIG_CFS_BANDWIDTH 9767 if (period_us_p) 9768 *period_us_p = tg_get_cfs_period(tg); 9769 if (quota_us_p) 9770 *quota_us_p = tg_get_cfs_quota(tg); 9771 if (burst_us_p) 9772 *burst_us_p = tg_get_cfs_burst(tg); 9773 #else /* !CONFIG_CFS_BANDWIDTH */ 9774 if (period_us_p) 9775 *period_us_p = tg->scx.bw_period_us; 9776 if (quota_us_p) 9777 *quota_us_p = tg->scx.bw_quota_us; 9778 if (burst_us_p) 9779 *burst_us_p = tg->scx.bw_burst_us; 9780 #endif /* CONFIG_CFS_BANDWIDTH */ 9781 } 9782 9783 static u64 cpu_period_read_u64(struct cgroup_subsys_state *css, 9784 struct cftype *cft) 9785 { 9786 u64 period_us; 9787 9788 tg_bandwidth(css_tg(css), &period_us, NULL, NULL); 9789 return period_us; 9790 } 9791 9792 static int tg_set_bandwidth(struct task_group *tg, 9793 u64 period_us, u64 quota_us, u64 burst_us) 9794 { 9795 const u64 max_usec = U64_MAX / NSEC_PER_USEC; 9796 int ret = 0; 9797 9798 if (tg == &root_task_group) 9799 return -EINVAL; 9800 9801 /* Values should survive translation to nsec */ 9802 if (period_us > max_usec || 9803 (quota_us != RUNTIME_INF && quota_us > max_usec) || 9804 burst_us > max_usec) 9805 return -EINVAL; 9806 9807 /* 9808 * Ensure we have some amount of bandwidth every period. This is to 9809 * prevent reaching a state of large arrears when throttled via 9810 * entity_tick() resulting in prolonged exit starvation. 9811 */ 9812 if (quota_us < min_bw_quota_period_us || 9813 period_us < min_bw_quota_period_us) 9814 return -EINVAL; 9815 9816 /* 9817 * Likewise, bound things on the other side by preventing insane quota 9818 * periods. This also allows us to normalize in computing quota 9819 * feasibility. 9820 */ 9821 if (period_us > max_bw_quota_period_us) 9822 return -EINVAL; 9823 9824 /* 9825 * Bound quota to defend quota against overflow during bandwidth shift. 9826 */ 9827 if (quota_us != RUNTIME_INF && quota_us > max_bw_runtime_us) 9828 return -EINVAL; 9829 9830 if (quota_us != RUNTIME_INF && (burst_us > quota_us || 9831 burst_us + quota_us > max_bw_runtime_us)) 9832 return -EINVAL; 9833 9834 #ifdef CONFIG_CFS_BANDWIDTH 9835 ret = tg_set_cfs_bandwidth(tg, period_us, quota_us, burst_us); 9836 #endif /* CONFIG_CFS_BANDWIDTH */ 9837 if (!ret) 9838 scx_group_set_bandwidth(tg, period_us, quota_us, burst_us); 9839 return ret; 9840 } 9841 9842 static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css, 9843 struct cftype *cft) 9844 { 9845 u64 quota_us; 9846 9847 tg_bandwidth(css_tg(css), NULL, "a_us, NULL); 9848 return quota_us; /* (s64)RUNTIME_INF becomes -1 */ 9849 } 9850 9851 static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css, 9852 struct cftype *cft) 9853 { 9854 u64 burst_us; 9855 9856 tg_bandwidth(css_tg(css), NULL, NULL, &burst_us); 9857 return burst_us; 9858 } 9859 9860 static int cpu_period_write_u64(struct cgroup_subsys_state *css, 9861 struct cftype *cftype, u64 period_us) 9862 { 9863 struct task_group *tg = css_tg(css); 9864 u64 quota_us, burst_us; 9865 9866 tg_bandwidth(tg, NULL, "a_us, &burst_us); 9867 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9868 } 9869 9870 static int cpu_quota_write_s64(struct cgroup_subsys_state *css, 9871 struct cftype *cftype, s64 quota_us) 9872 { 9873 struct task_group *tg = css_tg(css); 9874 u64 period_us, burst_us; 9875 9876 if (quota_us < 0) 9877 quota_us = RUNTIME_INF; 9878 9879 tg_bandwidth(tg, &period_us, NULL, &burst_us); 9880 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9881 } 9882 9883 static int cpu_burst_write_u64(struct cgroup_subsys_state *css, 9884 struct cftype *cftype, u64 burst_us) 9885 { 9886 struct task_group *tg = css_tg(css); 9887 u64 period_us, quota_us; 9888 9889 tg_bandwidth(tg, &period_us, "a_us, NULL); 9890 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9891 } 9892 #endif /* CONFIG_GROUP_SCHED_BANDWIDTH */ 9893 9894 #ifdef CONFIG_RT_GROUP_SCHED 9895 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 9896 struct cftype *cft, s64 val) 9897 { 9898 return sched_group_set_rt_runtime(css_tg(css), val); 9899 } 9900 9901 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 9902 struct cftype *cft) 9903 { 9904 return sched_group_rt_runtime(css_tg(css)); 9905 } 9906 9907 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 9908 struct cftype *cftype, u64 rt_period_us) 9909 { 9910 return sched_group_set_rt_period(css_tg(css), rt_period_us); 9911 } 9912 9913 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 9914 struct cftype *cft) 9915 { 9916 return sched_group_rt_period(css_tg(css)); 9917 } 9918 #endif /* CONFIG_RT_GROUP_SCHED */ 9919 9920 #ifdef CONFIG_GROUP_SCHED_WEIGHT 9921 static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, 9922 struct cftype *cft) 9923 { 9924 return css_tg(css)->idle; 9925 } 9926 9927 static int cpu_idle_write_s64(struct cgroup_subsys_state *css, 9928 struct cftype *cft, s64 idle) 9929 { 9930 int ret; 9931 9932 ret = sched_group_set_idle(css_tg(css), idle); 9933 if (!ret) 9934 scx_group_set_idle(css_tg(css), idle); 9935 return ret; 9936 } 9937 #endif /* CONFIG_GROUP_SCHED_WEIGHT */ 9938 9939 static struct cftype cpu_legacy_files[] = { 9940 #ifdef CONFIG_GROUP_SCHED_WEIGHT 9941 { 9942 .name = "shares", 9943 .read_u64 = cpu_shares_read_u64, 9944 .write_u64 = cpu_shares_write_u64, 9945 }, 9946 { 9947 .name = "idle", 9948 .read_s64 = cpu_idle_read_s64, 9949 .write_s64 = cpu_idle_write_s64, 9950 }, 9951 #endif 9952 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH 9953 { 9954 .name = "cfs_period_us", 9955 .read_u64 = cpu_period_read_u64, 9956 .write_u64 = cpu_period_write_u64, 9957 }, 9958 { 9959 .name = "cfs_quota_us", 9960 .read_s64 = cpu_quota_read_s64, 9961 .write_s64 = cpu_quota_write_s64, 9962 }, 9963 { 9964 .name = "cfs_burst_us", 9965 .read_u64 = cpu_burst_read_u64, 9966 .write_u64 = cpu_burst_write_u64, 9967 }, 9968 #endif 9969 #ifdef CONFIG_CFS_BANDWIDTH 9970 { 9971 .name = "stat", 9972 .seq_show = cpu_cfs_stat_show, 9973 }, 9974 { 9975 .name = "stat.local", 9976 .seq_show = cpu_cfs_local_stat_show, 9977 }, 9978 #endif 9979 #ifdef CONFIG_UCLAMP_TASK_GROUP 9980 { 9981 .name = "uclamp.min", 9982 .flags = CFTYPE_NOT_ON_ROOT, 9983 .seq_show = cpu_uclamp_min_show, 9984 .write = cpu_uclamp_min_write, 9985 }, 9986 { 9987 .name = "uclamp.max", 9988 .flags = CFTYPE_NOT_ON_ROOT, 9989 .seq_show = cpu_uclamp_max_show, 9990 .write = cpu_uclamp_max_write, 9991 }, 9992 #endif 9993 { } /* Terminate */ 9994 }; 9995 9996 #ifdef CONFIG_RT_GROUP_SCHED 9997 static struct cftype rt_group_files[] = { 9998 { 9999 .name = "rt_runtime_us", 10000 .read_s64 = cpu_rt_runtime_read, 10001 .write_s64 = cpu_rt_runtime_write, 10002 }, 10003 { 10004 .name = "rt_period_us", 10005 .read_u64 = cpu_rt_period_read_uint, 10006 .write_u64 = cpu_rt_period_write_uint, 10007 }, 10008 { } /* Terminate */ 10009 }; 10010 10011 # ifdef CONFIG_RT_GROUP_SCHED_DEFAULT_DISABLED 10012 DEFINE_STATIC_KEY_FALSE(rt_group_sched); 10013 # else 10014 DEFINE_STATIC_KEY_TRUE(rt_group_sched); 10015 # endif 10016 10017 static int __init setup_rt_group_sched(char *str) 10018 { 10019 long val; 10020 10021 if (kstrtol(str, 0, &val) || val < 0 || val > 1) { 10022 pr_warn("Unable to set rt_group_sched\n"); 10023 return 1; 10024 } 10025 if (val) 10026 static_branch_enable(&rt_group_sched); 10027 else 10028 static_branch_disable(&rt_group_sched); 10029 10030 return 1; 10031 } 10032 __setup("rt_group_sched=", setup_rt_group_sched); 10033 10034 static int __init cpu_rt_group_init(void) 10035 { 10036 if (!rt_group_sched_enabled()) 10037 return 0; 10038 10039 WARN_ON(cgroup_add_legacy_cftypes(&cpu_cgrp_subsys, rt_group_files)); 10040 return 0; 10041 } 10042 subsys_initcall(cpu_rt_group_init); 10043 #endif /* CONFIG_RT_GROUP_SCHED */ 10044 10045 static int cpu_extra_stat_show(struct seq_file *sf, 10046 struct cgroup_subsys_state *css) 10047 { 10048 #ifdef CONFIG_CFS_BANDWIDTH 10049 { 10050 struct task_group *tg = css_tg(css); 10051 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10052 u64 throttled_usec, burst_usec; 10053 10054 throttled_usec = cfs_b->throttled_time; 10055 do_div(throttled_usec, NSEC_PER_USEC); 10056 burst_usec = cfs_b->burst_time; 10057 do_div(burst_usec, NSEC_PER_USEC); 10058 10059 seq_printf(sf, "nr_periods %d\n" 10060 "nr_throttled %d\n" 10061 "throttled_usec %llu\n" 10062 "nr_bursts %d\n" 10063 "burst_usec %llu\n", 10064 cfs_b->nr_periods, cfs_b->nr_throttled, 10065 throttled_usec, cfs_b->nr_burst, burst_usec); 10066 } 10067 #endif /* CONFIG_CFS_BANDWIDTH */ 10068 return 0; 10069 } 10070 10071 static int cpu_local_stat_show(struct seq_file *sf, 10072 struct cgroup_subsys_state *css) 10073 { 10074 #ifdef CONFIG_CFS_BANDWIDTH 10075 { 10076 struct task_group *tg = css_tg(css); 10077 u64 throttled_self_usec; 10078 10079 throttled_self_usec = throttled_time_self(tg); 10080 do_div(throttled_self_usec, NSEC_PER_USEC); 10081 10082 seq_printf(sf, "throttled_usec %llu\n", 10083 throttled_self_usec); 10084 } 10085 #endif 10086 return 0; 10087 } 10088 10089 #ifdef CONFIG_GROUP_SCHED_WEIGHT 10090 10091 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 10092 struct cftype *cft) 10093 { 10094 return sched_weight_to_cgroup(tg_weight(css_tg(css))); 10095 } 10096 10097 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 10098 struct cftype *cft, u64 cgrp_weight) 10099 { 10100 unsigned long weight; 10101 int ret; 10102 10103 if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX) 10104 return -ERANGE; 10105 10106 weight = sched_weight_from_cgroup(cgrp_weight); 10107 10108 ret = sched_group_set_shares(css_tg(css), scale_load(weight)); 10109 if (!ret) 10110 scx_group_set_weight(css_tg(css), cgrp_weight); 10111 return ret; 10112 } 10113 10114 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 10115 struct cftype *cft) 10116 { 10117 unsigned long weight = tg_weight(css_tg(css)); 10118 int last_delta = INT_MAX; 10119 int prio, delta; 10120 10121 /* find the closest nice value to the current weight */ 10122 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 10123 delta = abs(sched_prio_to_weight[prio] - weight); 10124 if (delta >= last_delta) 10125 break; 10126 last_delta = delta; 10127 } 10128 10129 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 10130 } 10131 10132 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 10133 struct cftype *cft, s64 nice) 10134 { 10135 unsigned long weight; 10136 int idx, ret; 10137 10138 if (nice < MIN_NICE || nice > MAX_NICE) 10139 return -ERANGE; 10140 10141 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 10142 idx = array_index_nospec(idx, 40); 10143 weight = sched_prio_to_weight[idx]; 10144 10145 ret = sched_group_set_shares(css_tg(css), scale_load(weight)); 10146 if (!ret) 10147 scx_group_set_weight(css_tg(css), 10148 sched_weight_to_cgroup(weight)); 10149 return ret; 10150 } 10151 #endif /* CONFIG_GROUP_SCHED_WEIGHT */ 10152 10153 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 10154 long period, long quota) 10155 { 10156 if (quota < 0) 10157 seq_puts(sf, "max"); 10158 else 10159 seq_printf(sf, "%ld", quota); 10160 10161 seq_printf(sf, " %ld\n", period); 10162 } 10163 10164 /* caller should put the current value in *@periodp before calling */ 10165 static int __maybe_unused cpu_period_quota_parse(char *buf, u64 *period_us_p, 10166 u64 *quota_us_p) 10167 { 10168 char tok[21]; /* U64_MAX */ 10169 10170 if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1) 10171 return -EINVAL; 10172 10173 if (sscanf(tok, "%llu", quota_us_p) < 1) { 10174 if (!strcmp(tok, "max")) 10175 *quota_us_p = RUNTIME_INF; 10176 else 10177 return -EINVAL; 10178 } 10179 10180 return 0; 10181 } 10182 10183 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH 10184 static int cpu_max_show(struct seq_file *sf, void *v) 10185 { 10186 struct task_group *tg = css_tg(seq_css(sf)); 10187 u64 period_us, quota_us; 10188 10189 tg_bandwidth(tg, &period_us, "a_us, NULL); 10190 cpu_period_quota_print(sf, period_us, quota_us); 10191 return 0; 10192 } 10193 10194 static ssize_t cpu_max_write(struct kernfs_open_file *of, 10195 char *buf, size_t nbytes, loff_t off) 10196 { 10197 struct task_group *tg = css_tg(of_css(of)); 10198 u64 period_us, quota_us, burst_us; 10199 int ret; 10200 10201 tg_bandwidth(tg, &period_us, NULL, &burst_us); 10202 ret = cpu_period_quota_parse(buf, &period_us, "a_us); 10203 if (!ret) 10204 ret = tg_set_bandwidth(tg, period_us, quota_us, burst_us); 10205 return ret ?: nbytes; 10206 } 10207 #endif /* CONFIG_CFS_BANDWIDTH */ 10208 10209 static struct cftype cpu_files[] = { 10210 #ifdef CONFIG_GROUP_SCHED_WEIGHT 10211 { 10212 .name = "weight", 10213 .flags = CFTYPE_NOT_ON_ROOT, 10214 .read_u64 = cpu_weight_read_u64, 10215 .write_u64 = cpu_weight_write_u64, 10216 }, 10217 { 10218 .name = "weight.nice", 10219 .flags = CFTYPE_NOT_ON_ROOT, 10220 .read_s64 = cpu_weight_nice_read_s64, 10221 .write_s64 = cpu_weight_nice_write_s64, 10222 }, 10223 { 10224 .name = "idle", 10225 .flags = CFTYPE_NOT_ON_ROOT, 10226 .read_s64 = cpu_idle_read_s64, 10227 .write_s64 = cpu_idle_write_s64, 10228 }, 10229 #endif 10230 #ifdef CONFIG_GROUP_SCHED_BANDWIDTH 10231 { 10232 .name = "max", 10233 .flags = CFTYPE_NOT_ON_ROOT, 10234 .seq_show = cpu_max_show, 10235 .write = cpu_max_write, 10236 }, 10237 { 10238 .name = "max.burst", 10239 .flags = CFTYPE_NOT_ON_ROOT, 10240 .read_u64 = cpu_burst_read_u64, 10241 .write_u64 = cpu_burst_write_u64, 10242 }, 10243 #endif /* CONFIG_CFS_BANDWIDTH */ 10244 #ifdef CONFIG_UCLAMP_TASK_GROUP 10245 { 10246 .name = "uclamp.min", 10247 .flags = CFTYPE_NOT_ON_ROOT, 10248 .seq_show = cpu_uclamp_min_show, 10249 .write = cpu_uclamp_min_write, 10250 }, 10251 { 10252 .name = "uclamp.max", 10253 .flags = CFTYPE_NOT_ON_ROOT, 10254 .seq_show = cpu_uclamp_max_show, 10255 .write = cpu_uclamp_max_write, 10256 }, 10257 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 10258 { } /* terminate */ 10259 }; 10260 10261 struct cgroup_subsys cpu_cgrp_subsys = { 10262 .css_alloc = cpu_cgroup_css_alloc, 10263 .css_online = cpu_cgroup_css_online, 10264 .css_offline = cpu_cgroup_css_offline, 10265 .css_released = cpu_cgroup_css_released, 10266 .css_free = cpu_cgroup_css_free, 10267 .css_extra_stat_show = cpu_extra_stat_show, 10268 .css_local_stat_show = cpu_local_stat_show, 10269 .can_attach = cpu_cgroup_can_attach, 10270 .attach = cpu_cgroup_attach, 10271 .cancel_attach = cpu_cgroup_cancel_attach, 10272 .legacy_cftypes = cpu_legacy_files, 10273 .dfl_cftypes = cpu_files, 10274 .early_init = true, 10275 .threaded = true, 10276 }; 10277 10278 #endif /* CONFIG_CGROUP_SCHED */ 10279 10280 void dump_cpu_task(int cpu) 10281 { 10282 if (in_hardirq() && cpu == smp_processor_id()) { 10283 struct pt_regs *regs; 10284 10285 regs = get_irq_regs(); 10286 if (regs) { 10287 show_regs(regs); 10288 return; 10289 } 10290 } 10291 10292 if (trigger_single_cpu_backtrace(cpu)) 10293 return; 10294 10295 pr_info("Task dump for CPU %d:\n", cpu); 10296 sched_show_task(cpu_curr(cpu)); 10297 } 10298 10299 /* 10300 * Nice levels are multiplicative, with a gentle 10% change for every 10301 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 10302 * nice 1, it will get ~10% less CPU time than another CPU-bound task 10303 * that remained on nice 0. 10304 * 10305 * The "10% effect" is relative and cumulative: from _any_ nice level, 10306 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 10307 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 10308 * If a task goes up by ~10% and another task goes down by ~10% then 10309 * the relative distance between them is ~25%.) 10310 */ 10311 const int sched_prio_to_weight[40] = { 10312 /* -20 */ 88761, 71755, 56483, 46273, 36291, 10313 /* -15 */ 29154, 23254, 18705, 14949, 11916, 10314 /* -10 */ 9548, 7620, 6100, 4904, 3906, 10315 /* -5 */ 3121, 2501, 1991, 1586, 1277, 10316 /* 0 */ 1024, 820, 655, 526, 423, 10317 /* 5 */ 335, 272, 215, 172, 137, 10318 /* 10 */ 110, 87, 70, 56, 45, 10319 /* 15 */ 36, 29, 23, 18, 15, 10320 }; 10321 10322 /* 10323 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, pre-calculated. 10324 * 10325 * In cases where the weight does not change often, we can use the 10326 * pre-calculated inverse to speed up arithmetics by turning divisions 10327 * into multiplications: 10328 */ 10329 const u32 sched_prio_to_wmult[40] = { 10330 /* -20 */ 48388, 59856, 76040, 92818, 118348, 10331 /* -15 */ 147320, 184698, 229616, 287308, 360437, 10332 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 10333 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 10334 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 10335 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 10336 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 10337 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 10338 }; 10339 10340 void call_trace_sched_update_nr_running(struct rq *rq, int count) 10341 { 10342 trace_sched_update_nr_running_tp(rq, count); 10343 } 10344 10345 #ifdef CONFIG_SCHED_MM_CID 10346 10347 /* 10348 * @cid_lock: Guarantee forward-progress of cid allocation. 10349 * 10350 * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock 10351 * is only used when contention is detected by the lock-free allocation so 10352 * forward progress can be guaranteed. 10353 */ 10354 DEFINE_RAW_SPINLOCK(cid_lock); 10355 10356 /* 10357 * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock. 10358 * 10359 * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is 10360 * detected, it is set to 1 to ensure that all newly coming allocations are 10361 * serialized by @cid_lock until the allocation which detected contention 10362 * completes and sets @use_cid_lock back to 0. This guarantees forward progress 10363 * of a cid allocation. 10364 */ 10365 int use_cid_lock; 10366 10367 /* 10368 * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid 10369 * concurrently with respect to the execution of the source runqueue context 10370 * switch. 10371 * 10372 * There is one basic properties we want to guarantee here: 10373 * 10374 * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively 10375 * used by a task. That would lead to concurrent allocation of the cid and 10376 * userspace corruption. 10377 * 10378 * Provide this guarantee by introducing a Dekker memory ordering to guarantee 10379 * that a pair of loads observe at least one of a pair of stores, which can be 10380 * shown as: 10381 * 10382 * X = Y = 0 10383 * 10384 * w[X]=1 w[Y]=1 10385 * MB MB 10386 * r[Y]=y r[X]=x 10387 * 10388 * Which guarantees that x==0 && y==0 is impossible. But rather than using 10389 * values 0 and 1, this algorithm cares about specific state transitions of the 10390 * runqueue current task (as updated by the scheduler context switch), and the 10391 * per-mm/cpu cid value. 10392 * 10393 * Let's introduce task (Y) which has task->mm == mm and task (N) which has 10394 * task->mm != mm for the rest of the discussion. There are two scheduler state 10395 * transitions on context switch we care about: 10396 * 10397 * (TSA) Store to rq->curr with transition from (N) to (Y) 10398 * 10399 * (TSB) Store to rq->curr with transition from (Y) to (N) 10400 * 10401 * On the remote-clear side, there is one transition we care about: 10402 * 10403 * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag 10404 * 10405 * There is also a transition to UNSET state which can be performed from all 10406 * sides (scheduler, remote-clear). It is always performed with a cmpxchg which 10407 * guarantees that only a single thread will succeed: 10408 * 10409 * (TMB) cmpxchg to *pcpu_cid to mark UNSET 10410 * 10411 * Just to be clear, what we do _not_ want to happen is a transition to UNSET 10412 * when a thread is actively using the cid (property (1)). 10413 * 10414 * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions. 10415 * 10416 * Scenario A) (TSA)+(TMA) (from next task perspective) 10417 * 10418 * CPU0 CPU1 10419 * 10420 * Context switch CS-1 Remote-clear 10421 * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA) 10422 * (implied barrier after cmpxchg) 10423 * - switch_mm_cid() 10424 * - memory barrier (see switch_mm_cid() 10425 * comment explaining how this barrier 10426 * is combined with other scheduler 10427 * barriers) 10428 * - mm_cid_get (next) 10429 * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr) 10430 * 10431 * This Dekker ensures that either task (Y) is observed by the 10432 * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are 10433 * observed. 10434 * 10435 * If task (Y) store is observed by rcu_dereference(), it means that there is 10436 * still an active task on the cpu. Remote-clear will therefore not transition 10437 * to UNSET, which fulfills property (1). 10438 * 10439 * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(), 10440 * it will move its state to UNSET, which clears the percpu cid perhaps 10441 * uselessly (which is not an issue for correctness). Because task (Y) is not 10442 * observed, CPU1 can move ahead to set the state to UNSET. Because moving 10443 * state to UNSET is done with a cmpxchg expecting that the old state has the 10444 * LAZY flag set, only one thread will successfully UNSET. 10445 * 10446 * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0 10447 * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and 10448 * CPU1 will observe task (Y) and do nothing more, which is fine. 10449 * 10450 * What we are effectively preventing with this Dekker is a scenario where 10451 * neither LAZY flag nor store (Y) are observed, which would fail property (1) 10452 * because this would UNSET a cid which is actively used. 10453 */ 10454 10455 void sched_mm_cid_migrate_from(struct task_struct *t) 10456 { 10457 t->migrate_from_cpu = task_cpu(t); 10458 } 10459 10460 static 10461 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, 10462 struct task_struct *t, 10463 struct mm_cid *src_pcpu_cid) 10464 { 10465 struct mm_struct *mm = t->mm; 10466 struct task_struct *src_task; 10467 int src_cid, last_mm_cid; 10468 10469 if (!mm) 10470 return -1; 10471 10472 last_mm_cid = t->last_mm_cid; 10473 /* 10474 * If the migrated task has no last cid, or if the current 10475 * task on src rq uses the cid, it means the source cid does not need 10476 * to be moved to the destination cpu. 10477 */ 10478 if (last_mm_cid == -1) 10479 return -1; 10480 src_cid = READ_ONCE(src_pcpu_cid->cid); 10481 if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid) 10482 return -1; 10483 10484 /* 10485 * If we observe an active task using the mm on this rq, it means we 10486 * are not the last task to be migrated from this cpu for this mm, so 10487 * there is no need to move src_cid to the destination cpu. 10488 */ 10489 guard(rcu)(); 10490 src_task = rcu_dereference(src_rq->curr); 10491 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 10492 t->last_mm_cid = -1; 10493 return -1; 10494 } 10495 10496 return src_cid; 10497 } 10498 10499 static 10500 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, 10501 struct task_struct *t, 10502 struct mm_cid *src_pcpu_cid, 10503 int src_cid) 10504 { 10505 struct task_struct *src_task; 10506 struct mm_struct *mm = t->mm; 10507 int lazy_cid; 10508 10509 if (src_cid == -1) 10510 return -1; 10511 10512 /* 10513 * Attempt to clear the source cpu cid to move it to the destination 10514 * cpu. 10515 */ 10516 lazy_cid = mm_cid_set_lazy_put(src_cid); 10517 if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid)) 10518 return -1; 10519 10520 /* 10521 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10522 * rq->curr->mm matches the scheduler barrier in context_switch() 10523 * between store to rq->curr and load of prev and next task's 10524 * per-mm/cpu cid. 10525 * 10526 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10527 * rq->curr->mm_cid_active matches the barrier in 10528 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 10529 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 10530 * load of per-mm/cpu cid. 10531 */ 10532 10533 /* 10534 * If we observe an active task using the mm on this rq after setting 10535 * the lazy-put flag, this task will be responsible for transitioning 10536 * from lazy-put flag set to MM_CID_UNSET. 10537 */ 10538 scoped_guard (rcu) { 10539 src_task = rcu_dereference(src_rq->curr); 10540 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 10541 /* 10542 * We observed an active task for this mm, there is therefore 10543 * no point in moving this cid to the destination cpu. 10544 */ 10545 t->last_mm_cid = -1; 10546 return -1; 10547 } 10548 } 10549 10550 /* 10551 * The src_cid is unused, so it can be unset. 10552 */ 10553 if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 10554 return -1; 10555 WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET); 10556 return src_cid; 10557 } 10558 10559 /* 10560 * Migration to dst cpu. Called with dst_rq lock held. 10561 * Interrupts are disabled, which keeps the window of cid ownership without the 10562 * source rq lock held small. 10563 */ 10564 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) 10565 { 10566 struct mm_cid *src_pcpu_cid, *dst_pcpu_cid; 10567 struct mm_struct *mm = t->mm; 10568 int src_cid, src_cpu; 10569 bool dst_cid_is_set; 10570 struct rq *src_rq; 10571 10572 lockdep_assert_rq_held(dst_rq); 10573 10574 if (!mm) 10575 return; 10576 src_cpu = t->migrate_from_cpu; 10577 if (src_cpu == -1) { 10578 t->last_mm_cid = -1; 10579 return; 10580 } 10581 /* 10582 * Move the src cid if the dst cid is unset. This keeps id 10583 * allocation closest to 0 in cases where few threads migrate around 10584 * many CPUs. 10585 * 10586 * If destination cid or recent cid is already set, we may have 10587 * to just clear the src cid to ensure compactness in frequent 10588 * migrations scenarios. 10589 * 10590 * It is not useful to clear the src cid when the number of threads is 10591 * greater or equal to the number of allowed CPUs, because user-space 10592 * can expect that the number of allowed cids can reach the number of 10593 * allowed CPUs. 10594 */ 10595 dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq)); 10596 dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) || 10597 !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid)); 10598 if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed)) 10599 return; 10600 src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu); 10601 src_rq = cpu_rq(src_cpu); 10602 src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid); 10603 if (src_cid == -1) 10604 return; 10605 src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid, 10606 src_cid); 10607 if (src_cid == -1) 10608 return; 10609 if (dst_cid_is_set) { 10610 __mm_cid_put(mm, src_cid); 10611 return; 10612 } 10613 /* Move src_cid to dst cpu. */ 10614 mm_cid_snapshot_time(dst_rq, mm); 10615 WRITE_ONCE(dst_pcpu_cid->cid, src_cid); 10616 WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid); 10617 } 10618 10619 static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid, 10620 int cpu) 10621 { 10622 struct rq *rq = cpu_rq(cpu); 10623 struct task_struct *t; 10624 int cid, lazy_cid; 10625 10626 cid = READ_ONCE(pcpu_cid->cid); 10627 if (!mm_cid_is_valid(cid)) 10628 return; 10629 10630 /* 10631 * Clear the cpu cid if it is set to keep cid allocation compact. If 10632 * there happens to be other tasks left on the source cpu using this 10633 * mm, the next task using this mm will reallocate its cid on context 10634 * switch. 10635 */ 10636 lazy_cid = mm_cid_set_lazy_put(cid); 10637 if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid)) 10638 return; 10639 10640 /* 10641 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10642 * rq->curr->mm matches the scheduler barrier in context_switch() 10643 * between store to rq->curr and load of prev and next task's 10644 * per-mm/cpu cid. 10645 * 10646 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 10647 * rq->curr->mm_cid_active matches the barrier in 10648 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 10649 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 10650 * load of per-mm/cpu cid. 10651 */ 10652 10653 /* 10654 * If we observe an active task using the mm on this rq after setting 10655 * the lazy-put flag, that task will be responsible for transitioning 10656 * from lazy-put flag set to MM_CID_UNSET. 10657 */ 10658 scoped_guard (rcu) { 10659 t = rcu_dereference(rq->curr); 10660 if (READ_ONCE(t->mm_cid_active) && t->mm == mm) 10661 return; 10662 } 10663 10664 /* 10665 * The cid is unused, so it can be unset. 10666 * Disable interrupts to keep the window of cid ownership without rq 10667 * lock small. 10668 */ 10669 scoped_guard (irqsave) { 10670 if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 10671 __mm_cid_put(mm, cid); 10672 } 10673 } 10674 10675 static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu) 10676 { 10677 struct rq *rq = cpu_rq(cpu); 10678 struct mm_cid *pcpu_cid; 10679 struct task_struct *curr; 10680 u64 rq_clock; 10681 10682 /* 10683 * rq->clock load is racy on 32-bit but one spurious clear once in a 10684 * while is irrelevant. 10685 */ 10686 rq_clock = READ_ONCE(rq->clock); 10687 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 10688 10689 /* 10690 * In order to take care of infrequently scheduled tasks, bump the time 10691 * snapshot associated with this cid if an active task using the mm is 10692 * observed on this rq. 10693 */ 10694 scoped_guard (rcu) { 10695 curr = rcu_dereference(rq->curr); 10696 if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) { 10697 WRITE_ONCE(pcpu_cid->time, rq_clock); 10698 return; 10699 } 10700 } 10701 10702 if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS) 10703 return; 10704 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 10705 } 10706 10707 static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu, 10708 int weight) 10709 { 10710 struct mm_cid *pcpu_cid; 10711 int cid; 10712 10713 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 10714 cid = READ_ONCE(pcpu_cid->cid); 10715 if (!mm_cid_is_valid(cid) || cid < weight) 10716 return; 10717 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 10718 } 10719 10720 static void task_mm_cid_work(struct callback_head *work) 10721 { 10722 unsigned long now = jiffies, old_scan, next_scan; 10723 struct task_struct *t = current; 10724 struct cpumask *cidmask; 10725 struct mm_struct *mm; 10726 int weight, cpu; 10727 10728 WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work)); 10729 10730 work->next = work; /* Prevent double-add */ 10731 if (t->flags & PF_EXITING) 10732 return; 10733 mm = t->mm; 10734 if (!mm) 10735 return; 10736 old_scan = READ_ONCE(mm->mm_cid_next_scan); 10737 next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY); 10738 if (!old_scan) { 10739 unsigned long res; 10740 10741 res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan); 10742 if (res != old_scan) 10743 old_scan = res; 10744 else 10745 old_scan = next_scan; 10746 } 10747 if (time_before(now, old_scan)) 10748 return; 10749 if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan)) 10750 return; 10751 cidmask = mm_cidmask(mm); 10752 /* Clear cids that were not recently used. */ 10753 for_each_possible_cpu(cpu) 10754 sched_mm_cid_remote_clear_old(mm, cpu); 10755 weight = cpumask_weight(cidmask); 10756 /* 10757 * Clear cids that are greater or equal to the cidmask weight to 10758 * recompact it. 10759 */ 10760 for_each_possible_cpu(cpu) 10761 sched_mm_cid_remote_clear_weight(mm, cpu, weight); 10762 } 10763 10764 void init_sched_mm_cid(struct task_struct *t) 10765 { 10766 struct mm_struct *mm = t->mm; 10767 int mm_users = 0; 10768 10769 if (mm) { 10770 mm_users = atomic_read(&mm->mm_users); 10771 if (mm_users == 1) 10772 mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY); 10773 } 10774 t->cid_work.next = &t->cid_work; /* Protect against double add */ 10775 init_task_work(&t->cid_work, task_mm_cid_work); 10776 } 10777 10778 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) 10779 { 10780 struct callback_head *work = &curr->cid_work; 10781 unsigned long now = jiffies; 10782 10783 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || 10784 work->next != work) 10785 return; 10786 if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan))) 10787 return; 10788 10789 /* No page allocation under rq lock */ 10790 task_work_add(curr, work, TWA_RESUME); 10791 } 10792 10793 void sched_mm_cid_exit_signals(struct task_struct *t) 10794 { 10795 struct mm_struct *mm = t->mm; 10796 struct rq *rq; 10797 10798 if (!mm) 10799 return; 10800 10801 preempt_disable(); 10802 rq = this_rq(); 10803 guard(rq_lock_irqsave)(rq); 10804 preempt_enable_no_resched(); /* holding spinlock */ 10805 WRITE_ONCE(t->mm_cid_active, 0); 10806 /* 10807 * Store t->mm_cid_active before loading per-mm/cpu cid. 10808 * Matches barrier in sched_mm_cid_remote_clear_old(). 10809 */ 10810 smp_mb(); 10811 mm_cid_put(mm); 10812 t->last_mm_cid = t->mm_cid = -1; 10813 } 10814 10815 void sched_mm_cid_before_execve(struct task_struct *t) 10816 { 10817 struct mm_struct *mm = t->mm; 10818 struct rq *rq; 10819 10820 if (!mm) 10821 return; 10822 10823 preempt_disable(); 10824 rq = this_rq(); 10825 guard(rq_lock_irqsave)(rq); 10826 preempt_enable_no_resched(); /* holding spinlock */ 10827 WRITE_ONCE(t->mm_cid_active, 0); 10828 /* 10829 * Store t->mm_cid_active before loading per-mm/cpu cid. 10830 * Matches barrier in sched_mm_cid_remote_clear_old(). 10831 */ 10832 smp_mb(); 10833 mm_cid_put(mm); 10834 t->last_mm_cid = t->mm_cid = -1; 10835 } 10836 10837 void sched_mm_cid_after_execve(struct task_struct *t) 10838 { 10839 struct mm_struct *mm = t->mm; 10840 struct rq *rq; 10841 10842 if (!mm) 10843 return; 10844 10845 preempt_disable(); 10846 rq = this_rq(); 10847 scoped_guard (rq_lock_irqsave, rq) { 10848 preempt_enable_no_resched(); /* holding spinlock */ 10849 WRITE_ONCE(t->mm_cid_active, 1); 10850 /* 10851 * Store t->mm_cid_active before loading per-mm/cpu cid. 10852 * Matches barrier in sched_mm_cid_remote_clear_old(). 10853 */ 10854 smp_mb(); 10855 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm); 10856 } 10857 } 10858 10859 void sched_mm_cid_fork(struct task_struct *t) 10860 { 10861 WARN_ON_ONCE(!t->mm || t->mm_cid != -1); 10862 t->mm_cid_active = 1; 10863 } 10864 #endif /* CONFIG_SCHED_MM_CID */ 10865 10866 static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx); 10867 10868 struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags) 10869 { 10870 struct sched_change_ctx *ctx = this_cpu_ptr(&sched_change_ctx); 10871 struct rq *rq = task_rq(p); 10872 10873 lockdep_assert_rq_held(rq); 10874 10875 *ctx = (struct sched_change_ctx){ 10876 .p = p, 10877 .flags = flags, 10878 .queued = task_on_rq_queued(p), 10879 .running = task_current_donor(rq, p), 10880 }; 10881 10882 if (ctx->queued) 10883 dequeue_task(rq, p, flags); 10884 if (ctx->running) 10885 put_prev_task(rq, p); 10886 10887 return ctx; 10888 } 10889 10890 void sched_change_end(struct sched_change_ctx *ctx) 10891 { 10892 struct task_struct *p = ctx->p; 10893 struct rq *rq = task_rq(p); 10894 10895 lockdep_assert_rq_held(rq); 10896 10897 if (ctx->queued) 10898 enqueue_task(rq, p, ctx->flags | ENQUEUE_NOCLOCK); 10899 if (ctx->running) 10900 set_next_task(rq, p); 10901 }