개념 설명 전체 · v6.18.37 / kernel/time/hrtimer.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2005-2006, Thomas Gleixner <[email protected]> 4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar 5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner 6 * 7 * High-resolution kernel timers 8 * 9 * In contrast to the low-resolution timeout API, aka timer wheel, 10 * hrtimers provide finer resolution and accuracy depending on system 11 * configuration and capabilities. 12 * 13 * Started by: Thomas Gleixner and Ingo Molnar 14 * 15 * Credits: 16 * Based on the original timer wheel code 17 * 18 * Help, testing, suggestions, bugfixes, improvements were 19 * provided by: 20 * 21 * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel 22 * et. al. 23 */ 24 25 #include <linux/cpu.h> 26 #include <linux/export.h> 27 #include <linux/percpu.h> 28 #include <linux/hrtimer.h> 29 #include <linux/notifier.h> 30 #include <linux/syscalls.h> 31 #include <linux/interrupt.h> 32 #include <linux/tick.h> 33 #include <linux/err.h> 34 #include <linux/debugobjects.h> 35 #include <linux/sched/signal.h> 36 #include <linux/sched/sysctl.h> 37 #include <linux/sched/rt.h> 38 #include <linux/sched/deadline.h> 39 #include <linux/sched/nohz.h> 40 #include <linux/sched/debug.h> 41 #include <linux/sched/isolation.h> 42 #include <linux/timer.h> 43 #include <linux/freezer.h> 44 #include <linux/compat.h> 45 46 #include <linux/uaccess.h> 47 48 #include <trace/events/timer.h> 49 50 #include "tick-internal.h" 51 52 /* 53 * Masks for selecting the soft and hard context timers from 54 * cpu_base->active 55 */ 56 #define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT) 57 #define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1) 58 #define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT) 59 #define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD) 60 61 static void retrigger_next_event(void *arg); 62 static ktime_t __hrtimer_cb_get_time(clockid_t clock_id); 63 64 /* 65 * The timer bases: 66 * 67 * There are more clockids than hrtimer bases. Thus, we index 68 * into the timer bases by the hrtimer_base_type enum. When trying 69 * to reach a base using a clockid, hrtimer_clockid_to_base() 70 * is used to convert from clockid to the proper hrtimer_base_type. 71 */ 72 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = 73 { 74 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock), 75 .clock_base = 76 { 77 { 78 .index = HRTIMER_BASE_MONOTONIC, 79 .clockid = CLOCK_MONOTONIC, 80 }, 81 { 82 .index = HRTIMER_BASE_REALTIME, 83 .clockid = CLOCK_REALTIME, 84 }, 85 { 86 .index = HRTIMER_BASE_BOOTTIME, 87 .clockid = CLOCK_BOOTTIME, 88 }, 89 { 90 .index = HRTIMER_BASE_TAI, 91 .clockid = CLOCK_TAI, 92 }, 93 { 94 .index = HRTIMER_BASE_MONOTONIC_SOFT, 95 .clockid = CLOCK_MONOTONIC, 96 }, 97 { 98 .index = HRTIMER_BASE_REALTIME_SOFT, 99 .clockid = CLOCK_REALTIME, 100 }, 101 { 102 .index = HRTIMER_BASE_BOOTTIME_SOFT, 103 .clockid = CLOCK_BOOTTIME, 104 }, 105 { 106 .index = HRTIMER_BASE_TAI_SOFT, 107 .clockid = CLOCK_TAI, 108 }, 109 }, 110 .csd = CSD_INIT(retrigger_next_event, NULL) 111 }; 112 113 static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base) 114 { 115 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) 116 return true; 117 else 118 return likely(base->online); 119 } 120 121 /* 122 * Functions and macros which are different for UP/SMP systems are kept in a 123 * single place 124 */ 125 #ifdef CONFIG_SMP 126 127 /* 128 * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base() 129 * such that hrtimer_callback_running() can unconditionally dereference 130 * timer->base->cpu_base 131 */ 132 static struct hrtimer_cpu_base migration_cpu_base = { 133 .clock_base = { { 134 .cpu_base = &migration_cpu_base, 135 .seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq, 136 &migration_cpu_base.lock), 137 }, }, 138 }; 139 140 #define migration_base migration_cpu_base.clock_base[0] 141 142 /* 143 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock 144 * means that all timers which are tied to this base via timer->base are 145 * locked, and the base itself is locked too. 146 * 147 * So __run_timers/migrate_timers can safely modify all timers which could 148 * be found on the lists/queues. 149 * 150 * When the timer's base is locked, and the timer removed from list, it is 151 * possible to set timer->base = &migration_base and drop the lock: the timer 152 * remains locked. 153 */ 154 static 155 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, 156 unsigned long *flags) 157 __acquires(&timer->base->lock) 158 { 159 struct hrtimer_clock_base *base; 160 161 for (;;) { 162 base = READ_ONCE(timer->base); 163 if (likely(base != &migration_base)) { 164 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); 165 if (likely(base == timer->base)) 166 return base; 167 /* The timer has migrated to another CPU: */ 168 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); 169 } 170 cpu_relax(); 171 } 172 } 173 174 /* 175 * Check if the elected target is suitable considering its next 176 * event and the hotplug state of the current CPU. 177 * 178 * If the elected target is remote and its next event is after the timer 179 * to queue, then a remote reprogram is necessary. However there is no 180 * guarantee the IPI handling the operation would arrive in time to meet 181 * the high resolution deadline. In this case the local CPU becomes a 182 * preferred target, unless it is offline. 183 * 184 * High and low resolution modes are handled the same way for simplicity. 185 * 186 * Called with cpu_base->lock of target cpu held. 187 */ 188 static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base, 189 struct hrtimer_cpu_base *new_cpu_base, 190 struct hrtimer_cpu_base *this_cpu_base) 191 { 192 ktime_t expires; 193 194 /* 195 * The local CPU clockevent can be reprogrammed. Also get_target_base() 196 * guarantees it is online. 197 */ 198 if (new_cpu_base == this_cpu_base) 199 return true; 200 201 /* 202 * The offline local CPU can't be the default target if the 203 * next remote target event is after this timer. Keep the 204 * elected new base. An IPI will be issued to reprogram 205 * it as a last resort. 206 */ 207 if (!hrtimer_base_is_online(this_cpu_base)) 208 return true; 209 210 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 211 212 return expires >= new_base->cpu_base->expires_next; 213 } 214 215 static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, int pinned) 216 { 217 if (!hrtimer_base_is_online(base)) { 218 int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER)); 219 220 return &per_cpu(hrtimer_bases, cpu); 221 } 222 223 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 224 if (static_branch_likely(&timers_migration_enabled) && !pinned) 225 return &per_cpu(hrtimer_bases, get_nohz_timer_target()); 226 #endif 227 return base; 228 } 229 230 /* 231 * We switch the timer base to a power-optimized selected CPU target, 232 * if: 233 * - NO_HZ_COMMON is enabled 234 * - timer migration is enabled 235 * - the timer callback is not running 236 * - the timer is not the first expiring timer on the new target 237 * 238 * If one of the above requirements is not fulfilled we move the timer 239 * to the current CPU or leave it on the previously assigned CPU if 240 * the timer callback is currently running. 241 */ 242 static inline struct hrtimer_clock_base * 243 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, 244 int pinned) 245 { 246 struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base; 247 struct hrtimer_clock_base *new_base; 248 int basenum = base->index; 249 250 this_cpu_base = this_cpu_ptr(&hrtimer_bases); 251 new_cpu_base = get_target_base(this_cpu_base, pinned); 252 again: 253 new_base = &new_cpu_base->clock_base[basenum]; 254 255 if (base != new_base) { 256 /* 257 * We are trying to move timer to new_base. 258 * However we can't change timer's base while it is running, 259 * so we keep it on the same CPU. No hassle vs. reprogramming 260 * the event source in the high resolution case. The softirq 261 * code will take care of this when the timer function has 262 * completed. There is no conflict as we hold the lock until 263 * the timer is enqueued. 264 */ 265 if (unlikely(hrtimer_callback_running(timer))) 266 return base; 267 268 /* See the comment in lock_hrtimer_base() */ 269 WRITE_ONCE(timer->base, &migration_base); 270 raw_spin_unlock(&base->cpu_base->lock); 271 raw_spin_lock(&new_base->cpu_base->lock); 272 273 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, 274 this_cpu_base)) { 275 raw_spin_unlock(&new_base->cpu_base->lock); 276 raw_spin_lock(&base->cpu_base->lock); 277 new_cpu_base = this_cpu_base; 278 WRITE_ONCE(timer->base, base); 279 goto again; 280 } 281 WRITE_ONCE(timer->base, new_base); 282 } else { 283 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { 284 new_cpu_base = this_cpu_base; 285 goto again; 286 } 287 } 288 return new_base; 289 } 290 291 #else /* CONFIG_SMP */ 292 293 static inline struct hrtimer_clock_base * 294 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 295 __acquires(&timer->base->cpu_base->lock) 296 { 297 struct hrtimer_clock_base *base = timer->base; 298 299 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); 300 301 return base; 302 } 303 304 # define switch_hrtimer_base(t, b, p) (b) 305 306 #endif /* !CONFIG_SMP */ 307 308 /* 309 * Functions for the union type storage format of ktime_t which are 310 * too large for inlining: 311 */ 312 #if BITS_PER_LONG < 64 313 /* 314 * Divide a ktime value by a nanosecond value 315 */ 316 s64 __ktime_divns(const ktime_t kt, s64 div) 317 { 318 int sft = 0; 319 s64 dclc; 320 u64 tmp; 321 322 dclc = ktime_to_ns(kt); 323 tmp = dclc < 0 ? -dclc : dclc; 324 325 /* Make sure the divisor is less than 2^32: */ 326 while (div >> 32) { 327 sft++; 328 div >>= 1; 329 } 330 tmp >>= sft; 331 do_div(tmp, (u32) div); 332 return dclc < 0 ? -tmp : tmp; 333 } 334 EXPORT_SYMBOL_GPL(__ktime_divns); 335 #endif /* BITS_PER_LONG >= 64 */ 336 337 /* 338 * Add two ktime values and do a safety check for overflow: 339 */ 340 ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) 341 { 342 ktime_t res = ktime_add_unsafe(lhs, rhs); 343 344 /* 345 * We use KTIME_SEC_MAX here, the maximum timeout which we can 346 * return to user space in a timespec: 347 */ 348 if (res < 0 || res < lhs || res < rhs) 349 res = ktime_set(KTIME_SEC_MAX, 0); 350 351 return res; 352 } 353 354 EXPORT_SYMBOL_GPL(ktime_add_safe); 355 356 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS 357 358 static const struct debug_obj_descr hrtimer_debug_descr; 359 360 static void *hrtimer_debug_hint(void *addr) 361 { 362 return ACCESS_PRIVATE((struct hrtimer *)addr, function); 363 } 364 365 /* 366 * fixup_init is called when: 367 * - an active object is initialized 368 */ 369 static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state) 370 { 371 struct hrtimer *timer = addr; 372 373 switch (state) { 374 case ODEBUG_STATE_ACTIVE: 375 hrtimer_cancel(timer); 376 debug_object_init(timer, &hrtimer_debug_descr); 377 return true; 378 default: 379 return false; 380 } 381 } 382 383 /* 384 * fixup_activate is called when: 385 * - an active object is activated 386 * - an unknown non-static object is activated 387 */ 388 static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state) 389 { 390 switch (state) { 391 case ODEBUG_STATE_ACTIVE: 392 WARN_ON(1); 393 fallthrough; 394 default: 395 return false; 396 } 397 } 398 399 /* 400 * fixup_free is called when: 401 * - an active object is freed 402 */ 403 static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state) 404 { 405 struct hrtimer *timer = addr; 406 407 switch (state) { 408 case ODEBUG_STATE_ACTIVE: 409 hrtimer_cancel(timer); 410 debug_object_free(timer, &hrtimer_debug_descr); 411 return true; 412 default: 413 return false; 414 } 415 } 416 417 static const struct debug_obj_descr hrtimer_debug_descr = { 418 .name = "hrtimer", 419 .debug_hint = hrtimer_debug_hint, 420 .fixup_init = hrtimer_fixup_init, 421 .fixup_activate = hrtimer_fixup_activate, 422 .fixup_free = hrtimer_fixup_free, 423 }; 424 425 static inline void debug_hrtimer_init(struct hrtimer *timer) 426 { 427 debug_object_init(timer, &hrtimer_debug_descr); 428 } 429 430 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) 431 { 432 debug_object_init_on_stack(timer, &hrtimer_debug_descr); 433 } 434 435 static inline void debug_hrtimer_activate(struct hrtimer *timer, 436 enum hrtimer_mode mode) 437 { 438 debug_object_activate(timer, &hrtimer_debug_descr); 439 } 440 441 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) 442 { 443 debug_object_deactivate(timer, &hrtimer_debug_descr); 444 } 445 446 void destroy_hrtimer_on_stack(struct hrtimer *timer) 447 { 448 debug_object_free(timer, &hrtimer_debug_descr); 449 } 450 EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack); 451 452 #else 453 454 static inline void debug_hrtimer_init(struct hrtimer *timer) { } 455 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { } 456 static inline void debug_hrtimer_activate(struct hrtimer *timer, 457 enum hrtimer_mode mode) { } 458 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } 459 #endif 460 461 static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode) 462 { 463 debug_hrtimer_init(timer); 464 trace_hrtimer_setup(timer, clockid, mode); 465 } 466 467 static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid, 468 enum hrtimer_mode mode) 469 { 470 debug_hrtimer_init_on_stack(timer); 471 trace_hrtimer_setup(timer, clockid, mode); 472 } 473 474 static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed) 475 { 476 debug_hrtimer_activate(timer, mode); 477 trace_hrtimer_start(timer, mode, was_armed); 478 } 479 480 static struct hrtimer_clock_base * 481 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active) 482 { 483 unsigned int idx; 484 485 if (!*active) 486 return NULL; 487 488 idx = __ffs(*active); 489 *active &= ~(1U << idx); 490 491 return &cpu_base->clock_base[idx]; 492 } 493 494 #define for_each_active_base(base, cpu_base, active) \ 495 while ((base = __next_base((cpu_base), &(active)))) 496 497 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base, 498 const struct hrtimer *exclude, 499 unsigned int active, 500 ktime_t expires_next) 501 { 502 struct hrtimer_clock_base *base; 503 ktime_t expires; 504 505 for_each_active_base(base, cpu_base, active) { 506 struct timerqueue_node *next; 507 struct hrtimer *timer; 508 509 next = timerqueue_getnext(&base->active); 510 timer = container_of(next, struct hrtimer, node); 511 if (timer == exclude) { 512 /* Get to the next timer in the queue. */ 513 next = timerqueue_iterate_next(next); 514 if (!next) 515 continue; 516 517 timer = container_of(next, struct hrtimer, node); 518 } 519 expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 520 if (expires < expires_next) { 521 expires_next = expires; 522 523 /* Skip cpu_base update if a timer is being excluded. */ 524 if (exclude) 525 continue; 526 527 if (timer->is_soft) 528 cpu_base->softirq_next_timer = timer; 529 else 530 cpu_base->next_timer = timer; 531 } 532 } 533 /* 534 * clock_was_set() might have changed base->offset of any of 535 * the clock bases so the result might be negative. Fix it up 536 * to prevent a false positive in clockevents_program_event(). 537 */ 538 if (expires_next < 0) 539 expires_next = 0; 540 return expires_next; 541 } 542 543 /* 544 * Recomputes cpu_base::*next_timer and returns the earliest expires_next 545 * but does not set cpu_base::*expires_next, that is done by 546 * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating 547 * cpu_base::*expires_next right away, reprogramming logic would no longer 548 * work. 549 * 550 * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases, 551 * those timers will get run whenever the softirq gets handled, at the end of 552 * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases. 553 * 554 * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases. 555 * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual 556 * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD. 557 * 558 * @active_mask must be one of: 559 * - HRTIMER_ACTIVE_ALL, 560 * - HRTIMER_ACTIVE_SOFT, or 561 * - HRTIMER_ACTIVE_HARD. 562 */ 563 static ktime_t 564 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask) 565 { 566 unsigned int active; 567 struct hrtimer *next_timer = NULL; 568 ktime_t expires_next = KTIME_MAX; 569 570 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { 571 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; 572 cpu_base->softirq_next_timer = NULL; 573 expires_next = __hrtimer_next_event_base(cpu_base, NULL, 574 active, KTIME_MAX); 575 576 next_timer = cpu_base->softirq_next_timer; 577 } 578 579 if (active_mask & HRTIMER_ACTIVE_HARD) { 580 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; 581 cpu_base->next_timer = next_timer; 582 expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, 583 expires_next); 584 } 585 586 return expires_next; 587 } 588 589 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base) 590 { 591 ktime_t expires_next, soft = KTIME_MAX; 592 593 /* 594 * If the soft interrupt has already been activated, ignore the 595 * soft bases. They will be handled in the already raised soft 596 * interrupt. 597 */ 598 if (!cpu_base->softirq_activated) { 599 soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT); 600 /* 601 * Update the soft expiry time. clock_settime() might have 602 * affected it. 603 */ 604 cpu_base->softirq_expires_next = soft; 605 } 606 607 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD); 608 /* 609 * If a softirq timer is expiring first, update cpu_base->next_timer 610 * and program the hardware with the soft expiry time. 611 */ 612 if (expires_next > soft) { 613 cpu_base->next_timer = cpu_base->softirq_next_timer; 614 expires_next = soft; 615 } 616 617 return expires_next; 618 } 619 620 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) 621 { 622 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; 623 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; 624 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 625 626 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 627 offs_real, offs_boot, offs_tai); 628 629 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 630 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 631 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 632 633 return now; 634 } 635 636 /* 637 * Is the high resolution mode active ? 638 */ 639 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) 640 { 641 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ? 642 cpu_base->hres_active : 0; 643 } 644 645 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, 646 struct hrtimer *next_timer, 647 ktime_t expires_next) 648 { 649 cpu_base->expires_next = expires_next; 650 651 /* 652 * If hres is not active, hardware does not have to be 653 * reprogrammed yet. 654 * 655 * If a hang was detected in the last timer interrupt then we 656 * leave the hang delay active in the hardware. We want the 657 * system to make progress. That also prevents the following 658 * scenario: 659 * T1 expires 50ms from now 660 * T2 expires 5s from now 661 * 662 * T1 is removed, so this code is called and would reprogram 663 * the hardware to 5s from now. Any hrtimer_start after that 664 * will not reprogram the hardware due to hang_detected being 665 * set. So we'd effectively block all timers until the T2 event 666 * fires. 667 */ 668 if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) 669 return; 670 671 tick_program_event(expires_next, 1); 672 } 673 674 /* 675 * Reprogram the event source with checking both queues for the 676 * next event 677 * Called with interrupts disabled and base->lock held 678 */ 679 static void 680 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) 681 { 682 ktime_t expires_next; 683 684 expires_next = hrtimer_update_next_event(cpu_base); 685 686 if (skip_equal && expires_next == cpu_base->expires_next) 687 return; 688 689 __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next); 690 } 691 692 /* High resolution timer related functions */ 693 #ifdef CONFIG_HIGH_RES_TIMERS 694 695 /* 696 * High resolution timer enabled ? 697 */ 698 static bool hrtimer_hres_enabled __read_mostly = true; 699 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC; 700 EXPORT_SYMBOL_GPL(hrtimer_resolution); 701 702 /* 703 * Enable / Disable high resolution mode 704 */ 705 static int __init setup_hrtimer_hres(char *str) 706 { 707 return (kstrtobool(str, &hrtimer_hres_enabled) == 0); 708 } 709 710 __setup("highres=", setup_hrtimer_hres); 711 712 /* 713 * hrtimer_high_res_enabled - query, if the highres mode is enabled 714 */ 715 static inline int hrtimer_is_hres_enabled(void) 716 { 717 return hrtimer_hres_enabled; 718 } 719 720 /* 721 * Switch to high resolution mode 722 */ 723 static void hrtimer_switch_to_hres(void) 724 { 725 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 726 727 if (tick_init_highres()) { 728 pr_warn("Could not switch to high resolution mode on CPU %u\n", 729 base->cpu); 730 return; 731 } 732 base->hres_active = 1; 733 hrtimer_resolution = HIGH_RES_NSEC; 734 735 tick_setup_sched_timer(true); 736 /* "Retrigger" the interrupt to get things going */ 737 retrigger_next_event(NULL); 738 } 739 740 #else 741 742 static inline int hrtimer_is_hres_enabled(void) { return 0; } 743 static inline void hrtimer_switch_to_hres(void) { } 744 745 #endif /* CONFIG_HIGH_RES_TIMERS */ 746 /* 747 * Retrigger next event is called after clock was set with interrupts 748 * disabled through an SMP function call or directly from low level 749 * resume code. 750 * 751 * This is only invoked when: 752 * - CONFIG_HIGH_RES_TIMERS is enabled. 753 * - CONFIG_NOHZ_COMMON is enabled 754 * 755 * For the other cases this function is empty and because the call sites 756 * are optimized out it vanishes as well, i.e. no need for lots of 757 * #ifdeffery. 758 */ 759 static void retrigger_next_event(void *arg) 760 { 761 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 762 763 /* 764 * When high resolution mode or nohz is active, then the offsets of 765 * CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the 766 * next tick will take care of that. 767 * 768 * If high resolution mode is active then the next expiring timer 769 * must be reevaluated and the clock event device reprogrammed if 770 * necessary. 771 * 772 * In the NOHZ case the update of the offset and the reevaluation 773 * of the next expiring timer is enough. The return from the SMP 774 * function call will take care of the reprogramming in case the 775 * CPU was in a NOHZ idle sleep. 776 * 777 * In periodic low resolution mode, the next softirq expiration 778 * must also be updated. 779 */ 780 raw_spin_lock(&base->lock); 781 hrtimer_update_base(base); 782 if (hrtimer_hres_active(base)) 783 hrtimer_force_reprogram(base, 0); 784 else 785 hrtimer_update_next_event(base); 786 raw_spin_unlock(&base->lock); 787 } 788 789 /* 790 * When a timer is enqueued and expires earlier than the already enqueued 791 * timers, we have to check, whether it expires earlier than the timer for 792 * which the clock event device was armed. 793 * 794 * Called with interrupts disabled and base->cpu_base.lock held 795 */ 796 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) 797 { 798 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 799 struct hrtimer_clock_base *base = timer->base; 800 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 801 802 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); 803 804 /* 805 * CLOCK_REALTIME timer might be requested with an absolute 806 * expiry time which is less than base->offset. Set it to 0. 807 */ 808 if (expires < 0) 809 expires = 0; 810 811 if (timer->is_soft) { 812 /* 813 * soft hrtimer could be started on a remote CPU. In this 814 * case softirq_expires_next needs to be updated on the 815 * remote CPU. The soft hrtimer will not expire before the 816 * first hard hrtimer on the remote CPU - 817 * hrtimer_check_target() prevents this case. 818 */ 819 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base; 820 821 if (timer_cpu_base->softirq_activated) 822 return; 823 824 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next)) 825 return; 826 827 timer_cpu_base->softirq_next_timer = timer; 828 timer_cpu_base->softirq_expires_next = expires; 829 830 if (!ktime_before(expires, timer_cpu_base->expires_next) || 831 !reprogram) 832 return; 833 } 834 835 /* 836 * If the timer is not on the current cpu, we cannot reprogram 837 * the other cpus clock event device. 838 */ 839 if (base->cpu_base != cpu_base) 840 return; 841 842 if (expires >= cpu_base->expires_next) 843 return; 844 845 /* 846 * If the hrtimer interrupt is running, then it will reevaluate the 847 * clock bases and reprogram the clock event device. 848 */ 849 if (cpu_base->in_hrtirq) 850 return; 851 852 cpu_base->next_timer = timer; 853 854 __hrtimer_reprogram(cpu_base, timer, expires); 855 } 856 857 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, 858 unsigned int active) 859 { 860 struct hrtimer_clock_base *base; 861 unsigned int seq; 862 ktime_t expires; 863 864 /* 865 * Update the base offsets unconditionally so the following 866 * checks whether the SMP function call is required works. 867 * 868 * The update is safe even when the remote CPU is in the hrtimer 869 * interrupt or the hrtimer soft interrupt and expiring affected 870 * bases. Either it will see the update before handling a base or 871 * it will see it when it finishes the processing and reevaluates 872 * the next expiring timer. 873 */ 874 seq = cpu_base->clock_was_set_seq; 875 hrtimer_update_base(cpu_base); 876 877 /* 878 * If the sequence did not change over the update then the 879 * remote CPU already handled it. 880 */ 881 if (seq == cpu_base->clock_was_set_seq) 882 return false; 883 884 /* 885 * If the remote CPU is currently handling an hrtimer interrupt, it 886 * will reevaluate the first expiring timer of all clock bases 887 * before reprogramming. Nothing to do here. 888 */ 889 if (cpu_base->in_hrtirq) 890 return false; 891 892 /* 893 * Walk the affected clock bases and check whether the first expiring 894 * timer in a clock base is moving ahead of the first expiring timer of 895 * @cpu_base. If so, the IPI must be invoked because per CPU clock 896 * event devices cannot be remotely reprogrammed. 897 */ 898 active &= cpu_base->active_bases; 899 900 for_each_active_base(base, cpu_base, active) { 901 struct timerqueue_node *next; 902 903 next = timerqueue_getnext(&base->active); 904 expires = ktime_sub(next->expires, base->offset); 905 if (expires < cpu_base->expires_next) 906 return true; 907 908 /* Extra check for softirq clock bases */ 909 if (base->index < HRTIMER_BASE_MONOTONIC_SOFT) 910 continue; 911 if (cpu_base->softirq_activated) 912 continue; 913 if (expires < cpu_base->softirq_expires_next) 914 return true; 915 } 916 return false; 917 } 918 919 /* 920 * Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and 921 * CLOCK_BOOTTIME (for late sleep time injection). 922 * 923 * This requires to update the offsets for these clocks 924 * vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this 925 * also requires to eventually reprogram the per CPU clock event devices 926 * when the change moves an affected timer ahead of the first expiring 927 * timer on that CPU. Obviously remote per CPU clock event devices cannot 928 * be reprogrammed. The other reason why an IPI has to be sent is when the 929 * system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets 930 * in the tick, which obviously might be stopped, so this has to bring out 931 * the remote CPU which might sleep in idle to get this sorted. 932 */ 933 void clock_was_set(unsigned int bases) 934 { 935 struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases); 936 cpumask_var_t mask; 937 int cpu; 938 939 if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active) 940 goto out_timerfd; 941 942 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 943 on_each_cpu(retrigger_next_event, NULL, 1); 944 goto out_timerfd; 945 } 946 947 /* Avoid interrupting CPUs if possible */ 948 cpus_read_lock(); 949 for_each_online_cpu(cpu) { 950 unsigned long flags; 951 952 cpu_base = &per_cpu(hrtimer_bases, cpu); 953 raw_spin_lock_irqsave(&cpu_base->lock, flags); 954 955 if (update_needs_ipi(cpu_base, bases)) 956 cpumask_set_cpu(cpu, mask); 957 958 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 959 } 960 961 preempt_disable(); 962 smp_call_function_many(mask, retrigger_next_event, NULL, 1); 963 preempt_enable(); 964 cpus_read_unlock(); 965 free_cpumask_var(mask); 966 967 out_timerfd: 968 timerfd_clock_was_set(); 969 } 970 971 static void clock_was_set_work(struct work_struct *work) 972 { 973 clock_was_set(CLOCK_SET_WALL); 974 } 975 976 static DECLARE_WORK(hrtimer_work, clock_was_set_work); 977 978 /* 979 * Called from timekeeping code to reprogram the hrtimer interrupt device 980 * on all cpus and to notify timerfd. 981 */ 982 void clock_was_set_delayed(void) 983 { 984 schedule_work(&hrtimer_work); 985 } 986 987 /* 988 * Called during resume either directly from via timekeeping_resume() 989 * or in the case of s2idle from tick_unfreeze() to ensure that the 990 * hrtimers are up to date. 991 */ 992 void hrtimers_resume_local(void) 993 { 994 lockdep_assert_irqs_disabled(); 995 /* Retrigger on the local CPU */ 996 retrigger_next_event(NULL); 997 } 998 999 /* 1000 * Counterpart to lock_hrtimer_base above: 1001 */ 1002 static inline 1003 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 1004 __releases(&timer->base->cpu_base->lock) 1005 { 1006 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); 1007 } 1008 1009 /** 1010 * hrtimer_forward() - forward the timer expiry 1011 * @timer: hrtimer to forward 1012 * @now: forward past this time 1013 * @interval: the interval to forward 1014 * 1015 * Forward the timer expiry so it will expire in the future. 1016 * 1017 * .. note:: 1018 * This only updates the timer expiry value and does not requeue the timer. 1019 * 1020 * There is also a variant of the function hrtimer_forward_now(). 1021 * 1022 * Context: Can be safely called from the callback function of @timer. If called 1023 * from other contexts @timer must neither be enqueued nor running the 1024 * callback and the caller needs to take care of serialization. 1025 * 1026 * Return: The number of overruns are returned. 1027 */ 1028 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) 1029 { 1030 u64 orun = 1; 1031 ktime_t delta; 1032 1033 delta = ktime_sub(now, hrtimer_get_expires(timer)); 1034 1035 if (delta < 0) 1036 return 0; 1037 1038 if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED)) 1039 return 0; 1040 1041 if (interval < hrtimer_resolution) 1042 interval = hrtimer_resolution; 1043 1044 if (unlikely(delta >= interval)) { 1045 s64 incr = ktime_to_ns(interval); 1046 1047 orun = ktime_divns(delta, incr); 1048 hrtimer_add_expires_ns(timer, incr * orun); 1049 if (hrtimer_get_expires_tv64(timer) > now) 1050 return orun; 1051 /* 1052 * This (and the ktime_add() below) is the 1053 * correction for exact: 1054 */ 1055 orun++; 1056 } 1057 hrtimer_add_expires(timer, interval); 1058 1059 return orun; 1060 } 1061 EXPORT_SYMBOL_GPL(hrtimer_forward); 1062 1063 /* 1064 * enqueue_hrtimer - internal function to (re)start a timer 1065 * 1066 * The timer is inserted in expiry order. Insertion into the 1067 * red black tree is O(log(n)). Must hold the base lock. 1068 * 1069 * Returns true when the new timer is the leftmost timer in the tree. 1070 */ 1071 static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, 1072 enum hrtimer_mode mode, bool was_armed) 1073 { 1074 debug_activate(timer, mode, was_armed); 1075 WARN_ON_ONCE(!base->cpu_base->online); 1076 1077 base->cpu_base->active_bases |= 1 << base->index; 1078 1079 /* Pairs with the lockless read in hrtimer_is_queued() */ 1080 WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED); 1081 1082 return timerqueue_add(&base->active, &timer->node); 1083 } 1084 1085 /* 1086 * __remove_hrtimer - internal function to remove a timer 1087 * 1088 * Caller must hold the base lock. 1089 * 1090 * High resolution timer mode reprograms the clock event device when the 1091 * timer is the one which expires next. The caller can disable this by setting 1092 * reprogram to zero. This is useful, when the context does a reprogramming 1093 * anyway (e.g. timer interrupt) 1094 */ 1095 static void __remove_hrtimer(struct hrtimer *timer, 1096 struct hrtimer_clock_base *base, 1097 u8 newstate, int reprogram) 1098 { 1099 struct hrtimer_cpu_base *cpu_base = base->cpu_base; 1100 u8 state = timer->state; 1101 1102 /* Pairs with the lockless read in hrtimer_is_queued() */ 1103 WRITE_ONCE(timer->state, newstate); 1104 if (!(state & HRTIMER_STATE_ENQUEUED)) 1105 return; 1106 1107 if (!timerqueue_del(&base->active, &timer->node)) 1108 cpu_base->active_bases &= ~(1 << base->index); 1109 1110 /* 1111 * Note: If reprogram is false we do not update 1112 * cpu_base->next_timer. This happens when we remove the first 1113 * timer on a remote cpu. No harm as we never dereference 1114 * cpu_base->next_timer. So the worst thing what can happen is 1115 * an superfluous call to hrtimer_force_reprogram() on the 1116 * remote cpu later on if the same timer gets enqueued again. 1117 */ 1118 if (reprogram && timer == cpu_base->next_timer) 1119 hrtimer_force_reprogram(cpu_base, 1); 1120 } 1121 1122 /* 1123 * remove hrtimer, called with base lock held 1124 */ 1125 static inline int 1126 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, 1127 bool restart, bool keep_local) 1128 { 1129 u8 state = timer->state; 1130 1131 if (state & HRTIMER_STATE_ENQUEUED) { 1132 bool reprogram; 1133 1134 debug_hrtimer_deactivate(timer); 1135 1136 /* 1137 * Remove the timer and force reprogramming when high 1138 * resolution mode is active and the timer is on the current 1139 * CPU. If we remove a timer on another CPU, reprogramming is 1140 * skipped. The interrupt event on this CPU is fired and 1141 * reprogramming happens in the interrupt handler. This is a 1142 * rare case and less expensive than a smp call. 1143 */ 1144 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 1145 1146 /* 1147 * If the timer is not restarted then reprogramming is 1148 * required if the timer is local. If it is local and about 1149 * to be restarted, avoid programming it twice (on removal 1150 * and a moment later when it's requeued). 1151 */ 1152 if (!restart) 1153 state = HRTIMER_STATE_INACTIVE; 1154 else 1155 reprogram &= !keep_local; 1156 1157 __remove_hrtimer(timer, base, state, reprogram); 1158 return 1; 1159 } 1160 return 0; 1161 } 1162 1163 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, 1164 const enum hrtimer_mode mode) 1165 { 1166 #ifdef CONFIG_TIME_LOW_RES 1167 /* 1168 * CONFIG_TIME_LOW_RES indicates that the system has no way to return 1169 * granular time values. For relative timers we add hrtimer_resolution 1170 * (i.e. one jiffy) to prevent short timeouts. 1171 */ 1172 timer->is_rel = mode & HRTIMER_MODE_REL; 1173 if (timer->is_rel) 1174 tim = ktime_add_safe(tim, hrtimer_resolution); 1175 #endif 1176 return tim; 1177 } 1178 1179 static void 1180 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram) 1181 { 1182 ktime_t expires; 1183 1184 /* 1185 * Find the next SOFT expiration. 1186 */ 1187 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT); 1188 1189 /* 1190 * reprogramming needs to be triggered, even if the next soft 1191 * hrtimer expires at the same time than the next hard 1192 * hrtimer. cpu_base->softirq_expires_next needs to be updated! 1193 */ 1194 if (expires == KTIME_MAX) 1195 return; 1196 1197 /* 1198 * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event() 1199 * cpu_base->*expires_next is only set by hrtimer_reprogram() 1200 */ 1201 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram); 1202 } 1203 1204 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, 1205 u64 delta_ns, const enum hrtimer_mode mode, 1206 struct hrtimer_clock_base *base) 1207 { 1208 struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases); 1209 struct hrtimer_clock_base *new_base; 1210 bool force_local, first, was_armed; 1211 1212 /* 1213 * If the timer is on the local cpu base and is the first expiring 1214 * timer then this might end up reprogramming the hardware twice 1215 * (on removal and on enqueue). To avoid that prevent the reprogram 1216 * on removal, keep the timer local to the current CPU and enforce 1217 * reprogramming after it is queued no matter whether it is the new 1218 * first expiring timer again or not. 1219 */ 1220 force_local = base->cpu_base == this_cpu_base; 1221 force_local &= base->cpu_base->next_timer == timer; 1222 1223 /* 1224 * Don't force local queuing if this enqueue happens on a unplugged 1225 * CPU after hrtimer_cpu_dying() has been invoked. 1226 */ 1227 force_local &= this_cpu_base->online; 1228 1229 /* 1230 * Remove an active timer from the queue. In case it is not queued 1231 * on the current CPU, make sure that remove_hrtimer() updates the 1232 * remote data correctly. 1233 * 1234 * If it's on the current CPU and the first expiring timer, then 1235 * skip reprogramming, keep the timer local and enforce 1236 * reprogramming later if it was the first expiring timer. This 1237 * avoids programming the underlying clock event twice (once at 1238 * removal and once after enqueue). 1239 */ 1240 was_armed = remove_hrtimer(timer, base, true, force_local); 1241 1242 if (mode & HRTIMER_MODE_REL) 1243 tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid)); 1244 1245 tim = hrtimer_update_lowres(timer, tim, mode); 1246 1247 hrtimer_set_expires_range_ns(timer, tim, delta_ns); 1248 1249 /* Switch the timer base, if necessary: */ 1250 if (!force_local) { 1251 new_base = switch_hrtimer_base(timer, base, 1252 mode & HRTIMER_MODE_PINNED); 1253 } else { 1254 new_base = base; 1255 } 1256 1257 first = enqueue_hrtimer(timer, new_base, mode, was_armed); 1258 1259 /* 1260 * If the hrtimer interrupt is running, then it will reevaluate the 1261 * clock bases and reprogram the clock event device. 1262 */ 1263 if (new_base->cpu_base->in_hrtirq) 1264 return false; 1265 1266 if (!force_local) { 1267 /* 1268 * If the current CPU base is online, then the timer is 1269 * never queued on a remote CPU if it would be the first 1270 * expiring timer there. 1271 */ 1272 if (hrtimer_base_is_online(this_cpu_base)) 1273 return first; 1274 1275 /* 1276 * Timer was enqueued remote because the current base is 1277 * already offline. If the timer is the first to expire, 1278 * kick the remote CPU to reprogram the clock event. 1279 */ 1280 if (first) { 1281 struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base; 1282 1283 smp_call_function_single_async(new_cpu_base->cpu, &new_cpu_base->csd); 1284 } 1285 return 0; 1286 } 1287 1288 /* 1289 * Timer was forced to stay on the current CPU to avoid 1290 * reprogramming on removal and enqueue. Force reprogram the 1291 * hardware by evaluating the new first expiring timer. 1292 */ 1293 hrtimer_force_reprogram(new_base->cpu_base, 1); 1294 return 0; 1295 } 1296 1297 /** 1298 * hrtimer_start_range_ns - (re)start an hrtimer 1299 * @timer: the timer to be added 1300 * @tim: expiry time 1301 * @delta_ns: "slack" range for the timer 1302 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or 1303 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); 1304 * softirq based mode is considered for debug purpose only! 1305 */ 1306 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, 1307 u64 delta_ns, const enum hrtimer_mode mode) 1308 { 1309 struct hrtimer_clock_base *base; 1310 unsigned long flags; 1311 1312 /* 1313 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft 1314 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard 1315 * expiry mode because unmarked timers are moved to softirq expiry. 1316 */ 1317 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 1318 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 1319 else 1320 WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); 1321 1322 base = lock_hrtimer_base(timer, &flags); 1323 1324 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) 1325 hrtimer_reprogram(timer, true); 1326 1327 unlock_hrtimer_base(timer, &flags); 1328 } 1329 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); 1330 1331 /** 1332 * hrtimer_try_to_cancel - try to deactivate a timer 1333 * @timer: hrtimer to stop 1334 * 1335 * Returns: 1336 * 1337 * * 0 when the timer was not active 1338 * * 1 when the timer was active 1339 * * -1 when the timer is currently executing the callback function and 1340 * cannot be stopped 1341 */ 1342 int hrtimer_try_to_cancel(struct hrtimer *timer) 1343 { 1344 struct hrtimer_clock_base *base; 1345 unsigned long flags; 1346 int ret = -1; 1347 1348 /* 1349 * Check lockless first. If the timer is not active (neither 1350 * enqueued nor running the callback, nothing to do here. The 1351 * base lock does not serialize against a concurrent enqueue, 1352 * so we can avoid taking it. 1353 */ 1354 if (!hrtimer_active(timer)) 1355 return 0; 1356 1357 base = lock_hrtimer_base(timer, &flags); 1358 1359 if (!hrtimer_callback_running(timer)) { 1360 ret = remove_hrtimer(timer, base, false, false); 1361 if (ret) 1362 trace_hrtimer_cancel(timer); 1363 } 1364 1365 unlock_hrtimer_base(timer, &flags); 1366 1367 return ret; 1368 1369 } 1370 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); 1371 1372 #ifdef CONFIG_PREEMPT_RT 1373 static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) 1374 { 1375 spin_lock_init(&base->softirq_expiry_lock); 1376 } 1377 1378 static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) 1379 __acquires(&base->softirq_expiry_lock) 1380 { 1381 spin_lock(&base->softirq_expiry_lock); 1382 } 1383 1384 static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) 1385 __releases(&base->softirq_expiry_lock) 1386 { 1387 spin_unlock(&base->softirq_expiry_lock); 1388 } 1389 1390 /* 1391 * The counterpart to hrtimer_cancel_wait_running(). 1392 * 1393 * If there is a waiter for cpu_base->expiry_lock, then it was waiting for 1394 * the timer callback to finish. Drop expiry_lock and reacquire it. That 1395 * allows the waiter to acquire the lock and make progress. 1396 */ 1397 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, 1398 unsigned long flags) 1399 { 1400 if (atomic_read(&cpu_base->timer_waiters)) { 1401 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1402 spin_unlock(&cpu_base->softirq_expiry_lock); 1403 spin_lock(&cpu_base->softirq_expiry_lock); 1404 raw_spin_lock_irq(&cpu_base->lock); 1405 } 1406 } 1407 1408 #ifdef CONFIG_SMP 1409 static __always_inline bool is_migration_base(struct hrtimer_clock_base *base) 1410 { 1411 return base == &migration_base; 1412 } 1413 #else 1414 static __always_inline bool is_migration_base(struct hrtimer_clock_base *base) 1415 { 1416 return false; 1417 } 1418 #endif 1419 1420 /* 1421 * This function is called on PREEMPT_RT kernels when the fast path 1422 * deletion of a timer failed because the timer callback function was 1423 * running. 1424 * 1425 * This prevents priority inversion: if the soft irq thread is preempted 1426 * in the middle of a timer callback, then calling hrtimer_cancel() can 1427 * lead to two issues: 1428 * 1429 * - If the caller is on a remote CPU then it has to spin wait for the timer 1430 * handler to complete. This can result in unbound priority inversion. 1431 * 1432 * - If the caller originates from the task which preempted the timer 1433 * handler on the same CPU, then spin waiting for the timer handler to 1434 * complete is never going to end. 1435 */ 1436 void hrtimer_cancel_wait_running(const struct hrtimer *timer) 1437 { 1438 /* Lockless read. Prevent the compiler from reloading it below */ 1439 struct hrtimer_clock_base *base = READ_ONCE(timer->base); 1440 1441 /* 1442 * Just relax if the timer expires in hard interrupt context or if 1443 * it is currently on the migration base. 1444 */ 1445 if (!timer->is_soft || is_migration_base(base)) { 1446 cpu_relax(); 1447 return; 1448 } 1449 1450 /* 1451 * Mark the base as contended and grab the expiry lock, which is 1452 * held by the softirq across the timer callback. Drop the lock 1453 * immediately so the softirq can expire the next timer. In theory 1454 * the timer could already be running again, but that's more than 1455 * unlikely and just causes another wait loop. 1456 */ 1457 atomic_inc(&base->cpu_base->timer_waiters); 1458 spin_lock_bh(&base->cpu_base->softirq_expiry_lock); 1459 atomic_dec(&base->cpu_base->timer_waiters); 1460 spin_unlock_bh(&base->cpu_base->softirq_expiry_lock); 1461 } 1462 #else 1463 static inline void 1464 hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { } 1465 static inline void 1466 hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { } 1467 static inline void 1468 hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { } 1469 static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, 1470 unsigned long flags) { } 1471 #endif 1472 1473 /** 1474 * hrtimer_cancel - cancel a timer and wait for the handler to finish. 1475 * @timer: the timer to be cancelled 1476 * 1477 * Returns: 1478 * 0 when the timer was not active 1479 * 1 when the timer was active 1480 */ 1481 int hrtimer_cancel(struct hrtimer *timer) 1482 { 1483 int ret; 1484 1485 do { 1486 ret = hrtimer_try_to_cancel(timer); 1487 1488 if (ret < 0) 1489 hrtimer_cancel_wait_running(timer); 1490 } while (ret < 0); 1491 return ret; 1492 } 1493 EXPORT_SYMBOL_GPL(hrtimer_cancel); 1494 1495 /** 1496 * __hrtimer_get_remaining - get remaining time for the timer 1497 * @timer: the timer to read 1498 * @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y 1499 */ 1500 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust) 1501 { 1502 unsigned long flags; 1503 ktime_t rem; 1504 1505 lock_hrtimer_base(timer, &flags); 1506 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust) 1507 rem = hrtimer_expires_remaining_adjusted(timer); 1508 else 1509 rem = hrtimer_expires_remaining(timer); 1510 unlock_hrtimer_base(timer, &flags); 1511 1512 return rem; 1513 } 1514 EXPORT_SYMBOL_GPL(__hrtimer_get_remaining); 1515 1516 #ifdef CONFIG_NO_HZ_COMMON 1517 /** 1518 * hrtimer_get_next_event - get the time until next expiry event 1519 * 1520 * Returns the next expiry time or KTIME_MAX if no timer is pending. 1521 */ 1522 u64 hrtimer_get_next_event(void) 1523 { 1524 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1525 u64 expires = KTIME_MAX; 1526 unsigned long flags; 1527 1528 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1529 1530 if (!hrtimer_hres_active(cpu_base)) 1531 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); 1532 1533 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1534 1535 return expires; 1536 } 1537 1538 /** 1539 * hrtimer_next_event_without - time until next expiry event w/o one timer 1540 * @exclude: timer to exclude 1541 * 1542 * Returns the next expiry time over all timers except for the @exclude one or 1543 * KTIME_MAX if none of them is pending. 1544 */ 1545 u64 hrtimer_next_event_without(const struct hrtimer *exclude) 1546 { 1547 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1548 u64 expires = KTIME_MAX; 1549 unsigned long flags; 1550 1551 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1552 1553 if (hrtimer_hres_active(cpu_base)) { 1554 unsigned int active; 1555 1556 if (!cpu_base->softirq_activated) { 1557 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; 1558 expires = __hrtimer_next_event_base(cpu_base, exclude, 1559 active, KTIME_MAX); 1560 } 1561 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; 1562 expires = __hrtimer_next_event_base(cpu_base, exclude, active, 1563 expires); 1564 } 1565 1566 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1567 1568 return expires; 1569 } 1570 #endif 1571 1572 static inline int hrtimer_clockid_to_base(clockid_t clock_id) 1573 { 1574 switch (clock_id) { 1575 case CLOCK_MONOTONIC: 1576 return HRTIMER_BASE_MONOTONIC; 1577 case CLOCK_REALTIME: 1578 return HRTIMER_BASE_REALTIME; 1579 case CLOCK_BOOTTIME: 1580 return HRTIMER_BASE_BOOTTIME; 1581 case CLOCK_TAI: 1582 return HRTIMER_BASE_TAI; 1583 default: 1584 WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id); 1585 return HRTIMER_BASE_MONOTONIC; 1586 } 1587 } 1588 1589 static ktime_t __hrtimer_cb_get_time(clockid_t clock_id) 1590 { 1591 switch (clock_id) { 1592 case CLOCK_MONOTONIC: 1593 return ktime_get(); 1594 case CLOCK_REALTIME: 1595 return ktime_get_real(); 1596 case CLOCK_BOOTTIME: 1597 return ktime_get_boottime(); 1598 case CLOCK_TAI: 1599 return ktime_get_clocktai(); 1600 default: 1601 WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id); 1602 return ktime_get(); 1603 } 1604 } 1605 1606 ktime_t hrtimer_cb_get_time(const struct hrtimer *timer) 1607 { 1608 return __hrtimer_cb_get_time(timer->base->clockid); 1609 } 1610 EXPORT_SYMBOL_GPL(hrtimer_cb_get_time); 1611 1612 static void __hrtimer_setup(struct hrtimer *timer, 1613 enum hrtimer_restart (*function)(struct hrtimer *), 1614 clockid_t clock_id, enum hrtimer_mode mode) 1615 { 1616 bool softtimer = !!(mode & HRTIMER_MODE_SOFT); 1617 struct hrtimer_cpu_base *cpu_base; 1618 int base; 1619 1620 /* 1621 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly 1622 * marked for hard interrupt expiry mode are moved into soft 1623 * interrupt context for latency reasons and because the callbacks 1624 * can invoke functions which might sleep on RT, e.g. spin_lock(). 1625 */ 1626 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD)) 1627 softtimer = true; 1628 1629 memset(timer, 0, sizeof(struct hrtimer)); 1630 1631 cpu_base = raw_cpu_ptr(&hrtimer_bases); 1632 1633 /* 1634 * POSIX magic: Relative CLOCK_REALTIME timers are not affected by 1635 * clock modifications, so they needs to become CLOCK_MONOTONIC to 1636 * ensure POSIX compliance. 1637 */ 1638 if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL) 1639 clock_id = CLOCK_MONOTONIC; 1640 1641 base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0; 1642 base += hrtimer_clockid_to_base(clock_id); 1643 timer->is_soft = softtimer; 1644 timer->is_hard = !!(mode & HRTIMER_MODE_HARD); 1645 timer->base = &cpu_base->clock_base[base]; 1646 timerqueue_init(&timer->node); 1647 1648 if (WARN_ON_ONCE(!function)) 1649 ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout; 1650 else 1651 ACCESS_PRIVATE(timer, function) = function; 1652 } 1653 1654 /** 1655 * hrtimer_setup - initialize a timer to the given clock 1656 * @timer: the timer to be initialized 1657 * @function: the callback function 1658 * @clock_id: the clock to be used 1659 * @mode: The modes which are relevant for initialization: 1660 * HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT, 1661 * HRTIMER_MODE_REL_SOFT 1662 * 1663 * The PINNED variants of the above can be handed in, 1664 * but the PINNED bit is ignored as pinning happens 1665 * when the hrtimer is started 1666 */ 1667 void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *), 1668 clockid_t clock_id, enum hrtimer_mode mode) 1669 { 1670 debug_setup(timer, clock_id, mode); 1671 __hrtimer_setup(timer, function, clock_id, mode); 1672 } 1673 EXPORT_SYMBOL_GPL(hrtimer_setup); 1674 1675 /** 1676 * hrtimer_setup_on_stack - initialize a timer on stack memory 1677 * @timer: The timer to be initialized 1678 * @function: the callback function 1679 * @clock_id: The clock to be used 1680 * @mode: The timer mode 1681 * 1682 * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack 1683 * memory. 1684 */ 1685 void hrtimer_setup_on_stack(struct hrtimer *timer, 1686 enum hrtimer_restart (*function)(struct hrtimer *), 1687 clockid_t clock_id, enum hrtimer_mode mode) 1688 { 1689 debug_setup_on_stack(timer, clock_id, mode); 1690 __hrtimer_setup(timer, function, clock_id, mode); 1691 } 1692 EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack); 1693 1694 /* 1695 * A timer is active, when it is enqueued into the rbtree or the 1696 * callback function is running or it's in the state of being migrated 1697 * to another cpu. 1698 * 1699 * It is important for this function to not return a false negative. 1700 */ 1701 bool hrtimer_active(const struct hrtimer *timer) 1702 { 1703 struct hrtimer_clock_base *base; 1704 unsigned int seq; 1705 1706 do { 1707 base = READ_ONCE(timer->base); 1708 seq = raw_read_seqcount_begin(&base->seq); 1709 1710 if (timer->state != HRTIMER_STATE_INACTIVE || 1711 base->running == timer) 1712 return true; 1713 1714 } while (read_seqcount_retry(&base->seq, seq) || 1715 base != READ_ONCE(timer->base)); 1716 1717 return false; 1718 } 1719 EXPORT_SYMBOL_GPL(hrtimer_active); 1720 1721 /* 1722 * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3 1723 * distinct sections: 1724 * 1725 * - queued: the timer is queued 1726 * - callback: the timer is being ran 1727 * - post: the timer is inactive or (re)queued 1728 * 1729 * On the read side we ensure we observe timer->state and cpu_base->running 1730 * from the same section, if anything changed while we looked at it, we retry. 1731 * This includes timer->base changing because sequence numbers alone are 1732 * insufficient for that. 1733 * 1734 * The sequence numbers are required because otherwise we could still observe 1735 * a false negative if the read side got smeared over multiple consecutive 1736 * __run_hrtimer() invocations. 1737 */ 1738 1739 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, 1740 struct hrtimer_clock_base *base, 1741 struct hrtimer *timer, ktime_t *now, 1742 unsigned long flags) __must_hold(&cpu_base->lock) 1743 { 1744 enum hrtimer_restart (*fn)(struct hrtimer *); 1745 bool expires_in_hardirq; 1746 int restart; 1747 1748 lockdep_assert_held(&cpu_base->lock); 1749 1750 debug_hrtimer_deactivate(timer); 1751 base->running = timer; 1752 1753 /* 1754 * Separate the ->running assignment from the ->state assignment. 1755 * 1756 * As with a regular write barrier, this ensures the read side in 1757 * hrtimer_active() cannot observe base->running == NULL && 1758 * timer->state == INACTIVE. 1759 */ 1760 raw_write_seqcount_barrier(&base->seq); 1761 1762 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0); 1763 fn = ACCESS_PRIVATE(timer, function); 1764 1765 /* 1766 * Clear the 'is relative' flag for the TIME_LOW_RES case. If the 1767 * timer is restarted with a period then it becomes an absolute 1768 * timer. If its not restarted it does not matter. 1769 */ 1770 if (IS_ENABLED(CONFIG_TIME_LOW_RES)) 1771 timer->is_rel = false; 1772 1773 /* 1774 * The timer is marked as running in the CPU base, so it is 1775 * protected against migration to a different CPU even if the lock 1776 * is dropped. 1777 */ 1778 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1779 trace_hrtimer_expire_entry(timer, now); 1780 expires_in_hardirq = lockdep_hrtimer_enter(timer); 1781 1782 restart = fn(timer); 1783 1784 lockdep_hrtimer_exit(expires_in_hardirq); 1785 trace_hrtimer_expire_exit(timer); 1786 raw_spin_lock_irq(&cpu_base->lock); 1787 1788 /* 1789 * Note: We clear the running state after enqueue_hrtimer and 1790 * we do not reprogram the event hardware. Happens either in 1791 * hrtimer_start_range_ns() or in hrtimer_interrupt() 1792 * 1793 * Note: Because we dropped the cpu_base->lock above, 1794 * hrtimer_start_range_ns() can have popped in and enqueued the timer 1795 * for us already. 1796 */ 1797 if (restart != HRTIMER_NORESTART && 1798 !(timer->state & HRTIMER_STATE_ENQUEUED)) 1799 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false); 1800 1801 /* 1802 * Separate the ->running assignment from the ->state assignment. 1803 * 1804 * As with a regular write barrier, this ensures the read side in 1805 * hrtimer_active() cannot observe base->running.timer == NULL && 1806 * timer->state == INACTIVE. 1807 */ 1808 raw_write_seqcount_barrier(&base->seq); 1809 1810 WARN_ON_ONCE(base->running != timer); 1811 base->running = NULL; 1812 } 1813 1814 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, 1815 unsigned long flags, unsigned int active_mask) 1816 { 1817 struct hrtimer_clock_base *base; 1818 unsigned int active = cpu_base->active_bases & active_mask; 1819 1820 for_each_active_base(base, cpu_base, active) { 1821 struct timerqueue_node *node; 1822 ktime_t basenow; 1823 1824 basenow = ktime_add(now, base->offset); 1825 1826 while ((node = timerqueue_getnext(&base->active))) { 1827 struct hrtimer *timer; 1828 1829 timer = container_of(node, struct hrtimer, node); 1830 1831 /* 1832 * The immediate goal for using the softexpires is 1833 * minimizing wakeups, not running timers at the 1834 * earliest interrupt after their soft expiration. 1835 * This allows us to avoid using a Priority Search 1836 * Tree, which can answer a stabbing query for 1837 * overlapping intervals and instead use the simple 1838 * BST we already have. 1839 * We don't add extra wakeups by delaying timers that 1840 * are right-of a not yet expired timer, because that 1841 * timer will have to trigger a wakeup anyway. 1842 */ 1843 if (basenow < hrtimer_get_softexpires_tv64(timer)) 1844 break; 1845 1846 __run_hrtimer(cpu_base, base, timer, &basenow, flags); 1847 if (active_mask == HRTIMER_ACTIVE_SOFT) 1848 hrtimer_sync_wait_running(cpu_base, flags); 1849 } 1850 } 1851 } 1852 1853 static __latent_entropy void hrtimer_run_softirq(void) 1854 { 1855 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1856 unsigned long flags; 1857 ktime_t now; 1858 1859 hrtimer_cpu_base_lock_expiry(cpu_base); 1860 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1861 1862 now = hrtimer_update_base(cpu_base); 1863 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT); 1864 1865 cpu_base->softirq_activated = 0; 1866 hrtimer_update_softirq_timer(cpu_base, true); 1867 1868 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1869 hrtimer_cpu_base_unlock_expiry(cpu_base); 1870 } 1871 1872 #ifdef CONFIG_HIGH_RES_TIMERS 1873 1874 /* 1875 * High resolution timer interrupt 1876 * Called with interrupts disabled 1877 */ 1878 void hrtimer_interrupt(struct clock_event_device *dev) 1879 { 1880 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1881 ktime_t expires_next, now, entry_time, delta; 1882 unsigned long flags; 1883 int retries = 0; 1884 1885 BUG_ON(!cpu_base->hres_active); 1886 cpu_base->nr_events++; 1887 dev->next_event = KTIME_MAX; 1888 1889 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1890 entry_time = now = hrtimer_update_base(cpu_base); 1891 retry: 1892 cpu_base->in_hrtirq = 1; 1893 /* 1894 * We set expires_next to KTIME_MAX here with cpu_base->lock 1895 * held to prevent that a timer is enqueued in our queue via 1896 * the migration code. This does not affect enqueueing of 1897 * timers which run their callback and need to be requeued on 1898 * this CPU. 1899 */ 1900 cpu_base->expires_next = KTIME_MAX; 1901 1902 if (!ktime_before(now, cpu_base->softirq_expires_next)) { 1903 cpu_base->softirq_expires_next = KTIME_MAX; 1904 cpu_base->softirq_activated = 1; 1905 raise_timer_softirq(HRTIMER_SOFTIRQ); 1906 } 1907 1908 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); 1909 1910 /* Reevaluate the clock bases for the [soft] next expiry */ 1911 expires_next = hrtimer_update_next_event(cpu_base); 1912 /* 1913 * Store the new expiry value so the migration code can verify 1914 * against it. 1915 */ 1916 cpu_base->expires_next = expires_next; 1917 cpu_base->in_hrtirq = 0; 1918 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1919 1920 /* Reprogramming necessary ? */ 1921 if (!tick_program_event(expires_next, 0)) { 1922 cpu_base->hang_detected = 0; 1923 return; 1924 } 1925 1926 /* 1927 * The next timer was already expired due to: 1928 * - tracing 1929 * - long lasting callbacks 1930 * - being scheduled away when running in a VM 1931 * 1932 * We need to prevent that we loop forever in the hrtimer 1933 * interrupt routine. We give it 3 attempts to avoid 1934 * overreacting on some spurious event. 1935 * 1936 * Acquire base lock for updating the offsets and retrieving 1937 * the current time. 1938 */ 1939 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1940 now = hrtimer_update_base(cpu_base); 1941 cpu_base->nr_retries++; 1942 if (++retries < 3) 1943 goto retry; 1944 /* 1945 * Give the system a chance to do something else than looping 1946 * here. We stored the entry time, so we know exactly how long 1947 * we spent here. We schedule the next event this amount of 1948 * time away. 1949 */ 1950 cpu_base->nr_hangs++; 1951 cpu_base->hang_detected = 1; 1952 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1953 1954 delta = ktime_sub(now, entry_time); 1955 if ((unsigned int)delta > cpu_base->max_hang_time) 1956 cpu_base->max_hang_time = (unsigned int) delta; 1957 /* 1958 * Limit it to a sensible value as we enforce a longer 1959 * delay. Give the CPU at least 100ms to catch up. 1960 */ 1961 if (delta > 100 * NSEC_PER_MSEC) 1962 expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC); 1963 else 1964 expires_next = ktime_add(now, delta); 1965 tick_program_event(expires_next, 1); 1966 pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta)); 1967 } 1968 #endif /* !CONFIG_HIGH_RES_TIMERS */ 1969 1970 /* 1971 * Called from run_local_timers in hardirq context every jiffy 1972 */ 1973 void hrtimer_run_queues(void) 1974 { 1975 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1976 unsigned long flags; 1977 ktime_t now; 1978 1979 if (hrtimer_hres_active(cpu_base)) 1980 return; 1981 1982 /* 1983 * This _is_ ugly: We have to check periodically, whether we 1984 * can switch to highres and / or nohz mode. The clocksource 1985 * switch happens with xtime_lock held. Notification from 1986 * there only sets the check bit in the tick_oneshot code, 1987 * otherwise we might deadlock vs. xtime_lock. 1988 */ 1989 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) { 1990 hrtimer_switch_to_hres(); 1991 return; 1992 } 1993 1994 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1995 now = hrtimer_update_base(cpu_base); 1996 1997 if (!ktime_before(now, cpu_base->softirq_expires_next)) { 1998 cpu_base->softirq_expires_next = KTIME_MAX; 1999 cpu_base->softirq_activated = 1; 2000 raise_timer_softirq(HRTIMER_SOFTIRQ); 2001 } 2002 2003 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); 2004 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 2005 } 2006 2007 /* 2008 * Sleep related functions: 2009 */ 2010 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) 2011 { 2012 struct hrtimer_sleeper *t = 2013 container_of(timer, struct hrtimer_sleeper, timer); 2014 struct task_struct *task = t->task; 2015 2016 t->task = NULL; 2017 if (task) 2018 wake_up_process(task); 2019 2020 return HRTIMER_NORESTART; 2021 } 2022 2023 /** 2024 * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer 2025 * @sl: sleeper to be started 2026 * @mode: timer mode abs/rel 2027 * 2028 * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers 2029 * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context) 2030 */ 2031 void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, 2032 enum hrtimer_mode mode) 2033 { 2034 /* 2035 * Make the enqueue delivery mode check work on RT. If the sleeper 2036 * was initialized for hard interrupt delivery, force the mode bit. 2037 * This is a special case for hrtimer_sleepers because 2038 * __hrtimer_setup_sleeper() determines the delivery mode on RT so the 2039 * fiddling with this decision is avoided at the call sites. 2040 */ 2041 if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard) 2042 mode |= HRTIMER_MODE_HARD; 2043 2044 hrtimer_start_expires(&sl->timer, mode); 2045 } 2046 EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires); 2047 2048 static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, 2049 clockid_t clock_id, enum hrtimer_mode mode) 2050 { 2051 /* 2052 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly 2053 * marked for hard interrupt expiry mode are moved into soft 2054 * interrupt context either for latency reasons or because the 2055 * hrtimer callback takes regular spinlocks or invokes other 2056 * functions which are not suitable for hard interrupt context on 2057 * PREEMPT_RT. 2058 * 2059 * The hrtimer_sleeper callback is RT compatible in hard interrupt 2060 * context, but there is a latency concern: Untrusted userspace can 2061 * spawn many threads which arm timers for the same expiry time on 2062 * the same CPU. That causes a latency spike due to the wakeup of 2063 * a gazillion threads. 2064 * 2065 * OTOH, privileged real-time user space applications rely on the 2066 * low latency of hard interrupt wakeups. If the current task is in 2067 * a real-time scheduling class, mark the mode for hard interrupt 2068 * expiry. 2069 */ 2070 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 2071 if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT)) 2072 mode |= HRTIMER_MODE_HARD; 2073 } 2074 2075 __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode); 2076 sl->task = current; 2077 } 2078 2079 /** 2080 * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory 2081 * @sl: sleeper to be initialized 2082 * @clock_id: the clock to be used 2083 * @mode: timer mode abs/rel 2084 */ 2085 void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, 2086 clockid_t clock_id, enum hrtimer_mode mode) 2087 { 2088 debug_setup_on_stack(&sl->timer, clock_id, mode); 2089 __hrtimer_setup_sleeper(sl, clock_id, mode); 2090 } 2091 EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack); 2092 2093 int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts) 2094 { 2095 switch(restart->nanosleep.type) { 2096 #ifdef CONFIG_COMPAT_32BIT_TIME 2097 case TT_COMPAT: 2098 if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp)) 2099 return -EFAULT; 2100 break; 2101 #endif 2102 case TT_NATIVE: 2103 if (put_timespec64(ts, restart->nanosleep.rmtp)) 2104 return -EFAULT; 2105 break; 2106 default: 2107 BUG(); 2108 } 2109 return -ERESTART_RESTARTBLOCK; 2110 } 2111 2112 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) 2113 { 2114 struct restart_block *restart; 2115 2116 do { 2117 set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 2118 hrtimer_sleeper_start_expires(t, mode); 2119 2120 if (likely(t->task)) 2121 schedule(); 2122 2123 hrtimer_cancel(&t->timer); 2124 mode = HRTIMER_MODE_ABS; 2125 2126 } while (t->task && !signal_pending(current)); 2127 2128 __set_current_state(TASK_RUNNING); 2129 2130 if (!t->task) 2131 return 0; 2132 2133 restart = ¤t->restart_block; 2134 if (restart->nanosleep.type != TT_NONE) { 2135 ktime_t rem = hrtimer_expires_remaining(&t->timer); 2136 struct timespec64 rmt; 2137 2138 if (rem <= 0) 2139 return 0; 2140 rmt = ktime_to_timespec64(rem); 2141 2142 return nanosleep_copyout(restart, &rmt); 2143 } 2144 return -ERESTART_RESTARTBLOCK; 2145 } 2146 2147 static long __sched hrtimer_nanosleep_restart(struct restart_block *restart) 2148 { 2149 struct hrtimer_sleeper t; 2150 int ret; 2151 2152 hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS); 2153 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); 2154 ret = do_nanosleep(&t, HRTIMER_MODE_ABS); 2155 destroy_hrtimer_on_stack(&t.timer); 2156 return ret; 2157 } 2158 2159 long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, 2160 const clockid_t clockid) 2161 { 2162 struct restart_block *restart; 2163 struct hrtimer_sleeper t; 2164 int ret = 0; 2165 2166 hrtimer_setup_sleeper_on_stack(&t, clockid, mode); 2167 hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns); 2168 ret = do_nanosleep(&t, mode); 2169 if (ret != -ERESTART_RESTARTBLOCK) 2170 goto out; 2171 2172 /* Absolute timers do not update the rmtp value and restart: */ 2173 if (mode == HRTIMER_MODE_ABS) { 2174 ret = -ERESTARTNOHAND; 2175 goto out; 2176 } 2177 2178 restart = ¤t->restart_block; 2179 restart->nanosleep.clockid = t.timer.base->clockid; 2180 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); 2181 set_restart_fn(restart, hrtimer_nanosleep_restart); 2182 out: 2183 destroy_hrtimer_on_stack(&t.timer); 2184 return ret; 2185 } 2186 2187 #ifdef CONFIG_64BIT 2188 2189 SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp, 2190 struct __kernel_timespec __user *, rmtp) 2191 { 2192 struct timespec64 tu; 2193 2194 if (get_timespec64(&tu, rqtp)) 2195 return -EFAULT; 2196 2197 if (!timespec64_valid(&tu)) 2198 return -EINVAL; 2199 2200 current->restart_block.fn = do_no_restart_syscall; 2201 current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; 2202 current->restart_block.nanosleep.rmtp = rmtp; 2203 return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, 2204 CLOCK_MONOTONIC); 2205 } 2206 2207 #endif 2208 2209 #ifdef CONFIG_COMPAT_32BIT_TIME 2210 2211 SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp, 2212 struct old_timespec32 __user *, rmtp) 2213 { 2214 struct timespec64 tu; 2215 2216 if (get_old_timespec32(&tu, rqtp)) 2217 return -EFAULT; 2218 2219 if (!timespec64_valid(&tu)) 2220 return -EINVAL; 2221 2222 current->restart_block.fn = do_no_restart_syscall; 2223 current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; 2224 current->restart_block.nanosleep.compat_rmtp = rmtp; 2225 return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, 2226 CLOCK_MONOTONIC); 2227 } 2228 #endif 2229 2230 /* 2231 * Functions related to boot-time initialization: 2232 */ 2233 int hrtimers_prepare_cpu(unsigned int cpu) 2234 { 2235 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); 2236 int i; 2237 2238 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 2239 struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i]; 2240 2241 clock_b->cpu_base = cpu_base; 2242 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock); 2243 timerqueue_init_head(&clock_b->active); 2244 } 2245 2246 cpu_base->cpu = cpu; 2247 hrtimer_cpu_base_init_expiry_lock(cpu_base); 2248 return 0; 2249 } 2250 2251 int hrtimers_cpu_starting(unsigned int cpu) 2252 { 2253 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 2254 2255 /* Clear out any left over state from a CPU down operation */ 2256 cpu_base->active_bases = 0; 2257 cpu_base->hres_active = 0; 2258 cpu_base->hang_detected = 0; 2259 cpu_base->next_timer = NULL; 2260 cpu_base->softirq_next_timer = NULL; 2261 cpu_base->expires_next = KTIME_MAX; 2262 cpu_base->softirq_expires_next = KTIME_MAX; 2263 cpu_base->online = 1; 2264 return 0; 2265 } 2266 2267 #ifdef CONFIG_HOTPLUG_CPU 2268 2269 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, 2270 struct hrtimer_clock_base *new_base) 2271 { 2272 struct hrtimer *timer; 2273 struct timerqueue_node *node; 2274 2275 while ((node = timerqueue_getnext(&old_base->active))) { 2276 timer = container_of(node, struct hrtimer, node); 2277 BUG_ON(hrtimer_callback_running(timer)); 2278 debug_hrtimer_deactivate(timer); 2279 2280 /* 2281 * Mark it as ENQUEUED not INACTIVE otherwise the 2282 * timer could be seen as !active and just vanish away 2283 * under us on another CPU 2284 */ 2285 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); 2286 timer->base = new_base; 2287 /* 2288 * Enqueue the timers on the new cpu. This does not 2289 * reprogram the event device in case the timer 2290 * expires before the earliest on this CPU, but we run 2291 * hrtimer_interrupt after we migrated everything to 2292 * sort out already expired timers and reprogram the 2293 * event device. 2294 */ 2295 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true); 2296 } 2297 } 2298 2299 int hrtimers_cpu_dying(unsigned int dying_cpu) 2300 { 2301 int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER)); 2302 struct hrtimer_cpu_base *old_base, *new_base; 2303 2304 old_base = this_cpu_ptr(&hrtimer_bases); 2305 new_base = &per_cpu(hrtimer_bases, ncpu); 2306 2307 /* 2308 * The caller is globally serialized and nobody else 2309 * takes two locks at once, deadlock is not possible. 2310 */ 2311 raw_spin_lock(&old_base->lock); 2312 raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING); 2313 2314 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 2315 migrate_hrtimer_list(&old_base->clock_base[i], 2316 &new_base->clock_base[i]); 2317 } 2318 2319 /* Tell the other CPU to retrigger the next event */ 2320 smp_call_function_single(ncpu, retrigger_next_event, NULL, 0); 2321 2322 raw_spin_unlock(&new_base->lock); 2323 old_base->online = 0; 2324 raw_spin_unlock(&old_base->lock); 2325 2326 return 0; 2327 } 2328 2329 #endif /* CONFIG_HOTPLUG_CPU */ 2330 2331 void __init hrtimers_init(void) 2332 { 2333 hrtimers_prepare_cpu(smp_processor_id()); 2334 hrtimers_cpu_starting(smp_processor_id()); 2335 open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq); 2336 }