개념 설명 전체 · v6.6 / mm/compaction.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/compaction.c 4 * 5 * Memory compaction for the reduction of external fragmentation. Note that 6 * this heavily depends upon page migration to do all the real heavy 7 * lifting 8 * 9 * Copyright IBM Corp. 2007-2010 Mel Gorman <[email protected]> 10 */ 11 #include <linux/cpu.h> 12 #include <linux/swap.h> 13 #include <linux/migrate.h> 14 #include <linux/compaction.h> 15 #include <linux/mm_inline.h> 16 #include <linux/sched/signal.h> 17 #include <linux/backing-dev.h> 18 #include <linux/sysctl.h> 19 #include <linux/sysfs.h> 20 #include <linux/page-isolation.h> 21 #include <linux/kasan.h> 22 #include <linux/kthread.h> 23 #include <linux/freezer.h> 24 #include <linux/page_owner.h> 25 #include <linux/psi.h> 26 #include "internal.h" 27 28 #ifdef CONFIG_COMPACTION 29 /* 30 * Fragmentation score check interval for proactive compaction purposes. 31 */ 32 #define HPAGE_FRAG_CHECK_INTERVAL_MSEC (500) 33 34 static inline void count_compact_event(enum vm_event_item item) 35 { 36 count_vm_event(item); 37 } 38 39 static inline void count_compact_events(enum vm_event_item item, long delta) 40 { 41 count_vm_events(item, delta); 42 } 43 #else 44 #define count_compact_event(item) do { } while (0) 45 #define count_compact_events(item, delta) do { } while (0) 46 #endif 47 48 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 49 50 #define CREATE_TRACE_POINTS 51 #include <trace/events/compaction.h> 52 53 #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) 54 #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) 55 56 /* 57 * Page order with-respect-to which proactive compaction 58 * calculates external fragmentation, which is used as 59 * the "fragmentation score" of a node/zone. 60 */ 61 #if defined CONFIG_TRANSPARENT_HUGEPAGE 62 #define COMPACTION_HPAGE_ORDER HPAGE_PMD_ORDER 63 #elif defined CONFIG_HUGETLBFS 64 #define COMPACTION_HPAGE_ORDER HUGETLB_PAGE_ORDER 65 #else 66 #define COMPACTION_HPAGE_ORDER (PMD_SHIFT - PAGE_SHIFT) 67 #endif 68 69 static unsigned long release_freepages(struct list_head *freelist) 70 { 71 struct page *page, *next; 72 unsigned long high_pfn = 0; 73 74 list_for_each_entry_safe(page, next, freelist, lru) { 75 unsigned long pfn = page_to_pfn(page); 76 list_del(&page->lru); 77 __free_page(page); 78 if (pfn > high_pfn) 79 high_pfn = pfn; 80 } 81 82 return high_pfn; 83 } 84 85 static void split_map_pages(struct list_head *list) 86 { 87 unsigned int i, order, nr_pages; 88 struct page *page, *next; 89 LIST_HEAD(tmp_list); 90 91 list_for_each_entry_safe(page, next, list, lru) { 92 list_del(&page->lru); 93 94 order = page_private(page); 95 nr_pages = 1 << order; 96 97 post_alloc_hook(page, order, __GFP_MOVABLE); 98 if (order) 99 split_page(page, order); 100 101 for (i = 0; i < nr_pages; i++) { 102 list_add(&page->lru, &tmp_list); 103 page++; 104 } 105 } 106 107 list_splice(&tmp_list, list); 108 } 109 110 #ifdef CONFIG_COMPACTION 111 bool PageMovable(struct page *page) 112 { 113 const struct movable_operations *mops; 114 115 VM_BUG_ON_PAGE(!PageLocked(page), page); 116 if (!__PageMovable(page)) 117 return false; 118 119 mops = page_movable_ops(page); 120 if (mops) 121 return true; 122 123 return false; 124 } 125 126 void __SetPageMovable(struct page *page, const struct movable_operations *mops) 127 { 128 VM_BUG_ON_PAGE(!PageLocked(page), page); 129 VM_BUG_ON_PAGE((unsigned long)mops & PAGE_MAPPING_MOVABLE, page); 130 page->mapping = (void *)((unsigned long)mops | PAGE_MAPPING_MOVABLE); 131 } 132 EXPORT_SYMBOL(__SetPageMovable); 133 134 void __ClearPageMovable(struct page *page) 135 { 136 VM_BUG_ON_PAGE(!PageMovable(page), page); 137 /* 138 * This page still has the type of a movable page, but it's 139 * actually not movable any more. 140 */ 141 page->mapping = (void *)PAGE_MAPPING_MOVABLE; 142 } 143 EXPORT_SYMBOL(__ClearPageMovable); 144 145 /* Do not skip compaction more than 64 times */ 146 #define COMPACT_MAX_DEFER_SHIFT 6 147 148 /* 149 * Compaction is deferred when compaction fails to result in a page 150 * allocation success. 1 << compact_defer_shift, compactions are skipped up 151 * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT 152 */ 153 static void defer_compaction(struct zone *zone, int order) 154 { 155 zone->compact_considered = 0; 156 zone->compact_defer_shift++; 157 158 if (order < zone->compact_order_failed) 159 zone->compact_order_failed = order; 160 161 if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) 162 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; 163 164 trace_mm_compaction_defer_compaction(zone, order); 165 } 166 167 /* Returns true if compaction should be skipped this time */ 168 static bool compaction_deferred(struct zone *zone, int order) 169 { 170 unsigned long defer_limit = 1UL << zone->compact_defer_shift; 171 172 if (order < zone->compact_order_failed) 173 return false; 174 175 /* Avoid possible overflow */ 176 if (++zone->compact_considered >= defer_limit) { 177 zone->compact_considered = defer_limit; 178 return false; 179 } 180 181 trace_mm_compaction_deferred(zone, order); 182 183 return true; 184 } 185 186 /* 187 * Update defer tracking counters after successful compaction of given order, 188 * which means an allocation either succeeded (alloc_success == true) or is 189 * expected to succeed. 190 */ 191 void compaction_defer_reset(struct zone *zone, int order, 192 bool alloc_success) 193 { 194 if (alloc_success) { 195 zone->compact_considered = 0; 196 zone->compact_defer_shift = 0; 197 } 198 if (order >= zone->compact_order_failed) 199 zone->compact_order_failed = order + 1; 200 201 trace_mm_compaction_defer_reset(zone, order); 202 } 203 204 /* Returns true if restarting compaction after many failures */ 205 static bool compaction_restarting(struct zone *zone, int order) 206 { 207 if (order < zone->compact_order_failed) 208 return false; 209 210 return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && 211 zone->compact_considered >= 1UL << zone->compact_defer_shift; 212 } 213 214 /* Returns true if the pageblock should be scanned for pages to isolate. */ 215 static inline bool isolation_suitable(struct compact_control *cc, 216 struct page *page) 217 { 218 if (cc->ignore_skip_hint) 219 return true; 220 221 return !get_pageblock_skip(page); 222 } 223 224 static void reset_cached_positions(struct zone *zone) 225 { 226 zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; 227 zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; 228 zone->compact_cached_free_pfn = 229 pageblock_start_pfn(zone_end_pfn(zone) - 1); 230 } 231 232 #ifdef CONFIG_SPARSEMEM 233 /* 234 * If the PFN falls into an offline section, return the start PFN of the 235 * next online section. If the PFN falls into an online section or if 236 * there is no next online section, return 0. 237 */ 238 static unsigned long skip_offline_sections(unsigned long start_pfn) 239 { 240 unsigned long start_nr = pfn_to_section_nr(start_pfn); 241 242 if (online_section_nr(start_nr)) 243 return 0; 244 245 while (++start_nr <= __highest_present_section_nr) { 246 if (online_section_nr(start_nr)) 247 return section_nr_to_pfn(start_nr); 248 } 249 250 return 0; 251 } 252 253 /* 254 * If the PFN falls into an offline section, return the end PFN of the 255 * next online section in reverse. If the PFN falls into an online section 256 * or if there is no next online section in reverse, return 0. 257 */ 258 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn) 259 { 260 unsigned long start_nr = pfn_to_section_nr(start_pfn); 261 262 if (!start_nr || online_section_nr(start_nr)) 263 return 0; 264 265 while (start_nr-- > 0) { 266 if (online_section_nr(start_nr)) 267 return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION; 268 } 269 270 return 0; 271 } 272 #else 273 static unsigned long skip_offline_sections(unsigned long start_pfn) 274 { 275 return 0; 276 } 277 278 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn) 279 { 280 return 0; 281 } 282 #endif 283 284 /* 285 * Compound pages of >= pageblock_order should consistently be skipped until 286 * released. It is always pointless to compact pages of such order (if they are 287 * migratable), and the pageblocks they occupy cannot contain any free pages. 288 */ 289 static bool pageblock_skip_persistent(struct page *page) 290 { 291 if (!PageCompound(page)) 292 return false; 293 294 page = compound_head(page); 295 296 if (compound_order(page) >= pageblock_order) 297 return true; 298 299 return false; 300 } 301 302 static bool 303 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source, 304 bool check_target) 305 { 306 struct page *page = pfn_to_online_page(pfn); 307 struct page *block_page; 308 struct page *end_page; 309 unsigned long block_pfn; 310 311 if (!page) 312 return false; 313 if (zone != page_zone(page)) 314 return false; 315 if (pageblock_skip_persistent(page)) 316 return false; 317 318 /* 319 * If skip is already cleared do no further checking once the 320 * restart points have been set. 321 */ 322 if (check_source && check_target && !get_pageblock_skip(page)) 323 return true; 324 325 /* 326 * If clearing skip for the target scanner, do not select a 327 * non-movable pageblock as the starting point. 328 */ 329 if (!check_source && check_target && 330 get_pageblock_migratetype(page) != MIGRATE_MOVABLE) 331 return false; 332 333 /* Ensure the start of the pageblock or zone is online and valid */ 334 block_pfn = pageblock_start_pfn(pfn); 335 block_pfn = max(block_pfn, zone->zone_start_pfn); 336 block_page = pfn_to_online_page(block_pfn); 337 if (block_page) { 338 page = block_page; 339 pfn = block_pfn; 340 } 341 342 /* Ensure the end of the pageblock or zone is online and valid */ 343 block_pfn = pageblock_end_pfn(pfn) - 1; 344 block_pfn = min(block_pfn, zone_end_pfn(zone) - 1); 345 end_page = pfn_to_online_page(block_pfn); 346 if (!end_page) 347 return false; 348 349 /* 350 * Only clear the hint if a sample indicates there is either a 351 * free page or an LRU page in the block. One or other condition 352 * is necessary for the block to be a migration source/target. 353 */ 354 do { 355 if (check_source && PageLRU(page)) { 356 clear_pageblock_skip(page); 357 return true; 358 } 359 360 if (check_target && PageBuddy(page)) { 361 clear_pageblock_skip(page); 362 return true; 363 } 364 365 page += (1 << PAGE_ALLOC_COSTLY_ORDER); 366 } while (page <= end_page); 367 368 return false; 369 } 370 371 /* 372 * This function is called to clear all cached information on pageblocks that 373 * should be skipped for page isolation when the migrate and free page scanner 374 * meet. 375 */ 376 static void __reset_isolation_suitable(struct zone *zone) 377 { 378 unsigned long migrate_pfn = zone->zone_start_pfn; 379 unsigned long free_pfn = zone_end_pfn(zone) - 1; 380 unsigned long reset_migrate = free_pfn; 381 unsigned long reset_free = migrate_pfn; 382 bool source_set = false; 383 bool free_set = false; 384 385 if (!zone->compact_blockskip_flush) 386 return; 387 388 zone->compact_blockskip_flush = false; 389 390 /* 391 * Walk the zone and update pageblock skip information. Source looks 392 * for PageLRU while target looks for PageBuddy. When the scanner 393 * is found, both PageBuddy and PageLRU are checked as the pageblock 394 * is suitable as both source and target. 395 */ 396 for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages, 397 free_pfn -= pageblock_nr_pages) { 398 cond_resched(); 399 400 /* Update the migrate PFN */ 401 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) && 402 migrate_pfn < reset_migrate) { 403 source_set = true; 404 reset_migrate = migrate_pfn; 405 zone->compact_init_migrate_pfn = reset_migrate; 406 zone->compact_cached_migrate_pfn[0] = reset_migrate; 407 zone->compact_cached_migrate_pfn[1] = reset_migrate; 408 } 409 410 /* Update the free PFN */ 411 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) && 412 free_pfn > reset_free) { 413 free_set = true; 414 reset_free = free_pfn; 415 zone->compact_init_free_pfn = reset_free; 416 zone->compact_cached_free_pfn = reset_free; 417 } 418 } 419 420 /* Leave no distance if no suitable block was reset */ 421 if (reset_migrate >= reset_free) { 422 zone->compact_cached_migrate_pfn[0] = migrate_pfn; 423 zone->compact_cached_migrate_pfn[1] = migrate_pfn; 424 zone->compact_cached_free_pfn = free_pfn; 425 } 426 } 427 428 void reset_isolation_suitable(pg_data_t *pgdat) 429 { 430 int zoneid; 431 432 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 433 struct zone *zone = &pgdat->node_zones[zoneid]; 434 if (!populated_zone(zone)) 435 continue; 436 437 /* Only flush if a full compaction finished recently */ 438 if (zone->compact_blockskip_flush) 439 __reset_isolation_suitable(zone); 440 } 441 } 442 443 /* 444 * Sets the pageblock skip bit if it was clear. Note that this is a hint as 445 * locks are not required for read/writers. Returns true if it was already set. 446 */ 447 static bool test_and_set_skip(struct compact_control *cc, struct page *page) 448 { 449 bool skip; 450 451 /* Do not update if skip hint is being ignored */ 452 if (cc->ignore_skip_hint) 453 return false; 454 455 skip = get_pageblock_skip(page); 456 if (!skip && !cc->no_set_skip_hint) 457 set_pageblock_skip(page); 458 459 return skip; 460 } 461 462 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) 463 { 464 struct zone *zone = cc->zone; 465 466 /* Set for isolation rather than compaction */ 467 if (cc->no_set_skip_hint) 468 return; 469 470 pfn = pageblock_end_pfn(pfn); 471 472 /* Update where async and sync compaction should restart */ 473 if (pfn > zone->compact_cached_migrate_pfn[0]) 474 zone->compact_cached_migrate_pfn[0] = pfn; 475 if (cc->mode != MIGRATE_ASYNC && 476 pfn > zone->compact_cached_migrate_pfn[1]) 477 zone->compact_cached_migrate_pfn[1] = pfn; 478 } 479 480 /* 481 * If no pages were isolated then mark this pageblock to be skipped in the 482 * future. The information is later cleared by __reset_isolation_suitable(). 483 */ 484 static void update_pageblock_skip(struct compact_control *cc, 485 struct page *page, unsigned long pfn) 486 { 487 struct zone *zone = cc->zone; 488 489 if (cc->no_set_skip_hint) 490 return; 491 492 set_pageblock_skip(page); 493 494 if (pfn < zone->compact_cached_free_pfn) 495 zone->compact_cached_free_pfn = pfn; 496 } 497 #else 498 static inline bool isolation_suitable(struct compact_control *cc, 499 struct page *page) 500 { 501 return true; 502 } 503 504 static inline bool pageblock_skip_persistent(struct page *page) 505 { 506 return false; 507 } 508 509 static inline void update_pageblock_skip(struct compact_control *cc, 510 struct page *page, unsigned long pfn) 511 { 512 } 513 514 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) 515 { 516 } 517 518 static bool test_and_set_skip(struct compact_control *cc, struct page *page) 519 { 520 return false; 521 } 522 #endif /* CONFIG_COMPACTION */ 523 524 /* 525 * Compaction requires the taking of some coarse locks that are potentially 526 * very heavily contended. For async compaction, trylock and record if the 527 * lock is contended. The lock will still be acquired but compaction will 528 * abort when the current block is finished regardless of success rate. 529 * Sync compaction acquires the lock. 530 * 531 * Always returns true which makes it easier to track lock state in callers. 532 */ 533 static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags, 534 struct compact_control *cc) 535 __acquires(lock) 536 { 537 /* Track if the lock is contended in async mode */ 538 if (cc->mode == MIGRATE_ASYNC && !cc->contended) { 539 if (spin_trylock_irqsave(lock, *flags)) 540 return true; 541 542 cc->contended = true; 543 } 544 545 spin_lock_irqsave(lock, *flags); 546 return true; 547 } 548 549 /* 550 * Compaction requires the taking of some coarse locks that are potentially 551 * very heavily contended. The lock should be periodically unlocked to avoid 552 * having disabled IRQs for a long time, even when there is nobody waiting on 553 * the lock. It might also be that allowing the IRQs will result in 554 * need_resched() becoming true. If scheduling is needed, compaction schedules. 555 * Either compaction type will also abort if a fatal signal is pending. 556 * In either case if the lock was locked, it is dropped and not regained. 557 * 558 * Returns true if compaction should abort due to fatal signal pending. 559 * Returns false when compaction can continue. 560 */ 561 static bool compact_unlock_should_abort(spinlock_t *lock, 562 unsigned long flags, bool *locked, struct compact_control *cc) 563 { 564 if (*locked) { 565 spin_unlock_irqrestore(lock, flags); 566 *locked = false; 567 } 568 569 if (fatal_signal_pending(current)) { 570 cc->contended = true; 571 return true; 572 } 573 574 cond_resched(); 575 576 return false; 577 } 578 579 /* 580 * Isolate free pages onto a private freelist. If @strict is true, will abort 581 * returning 0 on any invalid PFNs or non-free pages inside of the pageblock 582 * (even though it may still end up isolating some pages). 583 */ 584 static unsigned long isolate_freepages_block(struct compact_control *cc, 585 unsigned long *start_pfn, 586 unsigned long end_pfn, 587 struct list_head *freelist, 588 unsigned int stride, 589 bool strict) 590 { 591 int nr_scanned = 0, total_isolated = 0; 592 struct page *page; 593 unsigned long flags = 0; 594 bool locked = false; 595 unsigned long blockpfn = *start_pfn; 596 unsigned int order; 597 598 /* Strict mode is for isolation, speed is secondary */ 599 if (strict) 600 stride = 1; 601 602 page = pfn_to_page(blockpfn); 603 604 /* Isolate free pages. */ 605 for (; blockpfn < end_pfn; blockpfn += stride, page += stride) { 606 int isolated; 607 608 /* 609 * Periodically drop the lock (if held) regardless of its 610 * contention, to give chance to IRQs. Abort if fatal signal 611 * pending. 612 */ 613 if (!(blockpfn % COMPACT_CLUSTER_MAX) 614 && compact_unlock_should_abort(&cc->zone->lock, flags, 615 &locked, cc)) 616 break; 617 618 nr_scanned++; 619 620 /* 621 * For compound pages such as THP and hugetlbfs, we can save 622 * potentially a lot of iterations if we skip them at once. 623 * The check is racy, but we can consider only valid values 624 * and the only danger is skipping too much. 625 */ 626 if (PageCompound(page)) { 627 const unsigned int order = compound_order(page); 628 629 if (likely(order <= MAX_ORDER)) { 630 blockpfn += (1UL << order) - 1; 631 page += (1UL << order) - 1; 632 nr_scanned += (1UL << order) - 1; 633 } 634 goto isolate_fail; 635 } 636 637 if (!PageBuddy(page)) 638 goto isolate_fail; 639 640 /* If we already hold the lock, we can skip some rechecking. */ 641 if (!locked) { 642 locked = compact_lock_irqsave(&cc->zone->lock, 643 &flags, cc); 644 645 /* Recheck this is a buddy page under lock */ 646 if (!PageBuddy(page)) 647 goto isolate_fail; 648 } 649 650 /* Found a free page, will break it into order-0 pages */ 651 order = buddy_order(page); 652 isolated = __isolate_free_page(page, order); 653 if (!isolated) 654 break; 655 set_page_private(page, order); 656 657 nr_scanned += isolated - 1; 658 total_isolated += isolated; 659 cc->nr_freepages += isolated; 660 list_add_tail(&page->lru, freelist); 661 662 if (!strict && cc->nr_migratepages <= cc->nr_freepages) { 663 blockpfn += isolated; 664 break; 665 } 666 /* Advance to the end of split page */ 667 blockpfn += isolated - 1; 668 page += isolated - 1; 669 continue; 670 671 isolate_fail: 672 if (strict) 673 break; 674 675 } 676 677 if (locked) 678 spin_unlock_irqrestore(&cc->zone->lock, flags); 679 680 /* 681 * There is a tiny chance that we have read bogus compound_order(), 682 * so be careful to not go outside of the pageblock. 683 */ 684 if (unlikely(blockpfn > end_pfn)) 685 blockpfn = end_pfn; 686 687 trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn, 688 nr_scanned, total_isolated); 689 690 /* Record how far we have got within the block */ 691 *start_pfn = blockpfn; 692 693 /* 694 * If strict isolation is requested by CMA then check that all the 695 * pages requested were isolated. If there were any failures, 0 is 696 * returned and CMA will fail. 697 */ 698 if (strict && blockpfn < end_pfn) 699 total_isolated = 0; 700 701 cc->total_free_scanned += nr_scanned; 702 if (total_isolated) 703 count_compact_events(COMPACTISOLATED, total_isolated); 704 return total_isolated; 705 } 706 707 /** 708 * isolate_freepages_range() - isolate free pages. 709 * @cc: Compaction control structure. 710 * @start_pfn: The first PFN to start isolating. 711 * @end_pfn: The one-past-last PFN. 712 * 713 * Non-free pages, invalid PFNs, or zone boundaries within the 714 * [start_pfn, end_pfn) range are considered errors, cause function to 715 * undo its actions and return zero. 716 * 717 * Otherwise, function returns one-past-the-last PFN of isolated page 718 * (which may be greater then end_pfn if end fell in a middle of 719 * a free page). 720 */ 721 unsigned long 722 isolate_freepages_range(struct compact_control *cc, 723 unsigned long start_pfn, unsigned long end_pfn) 724 { 725 unsigned long isolated, pfn, block_start_pfn, block_end_pfn; 726 LIST_HEAD(freelist); 727 728 pfn = start_pfn; 729 block_start_pfn = pageblock_start_pfn(pfn); 730 if (block_start_pfn < cc->zone->zone_start_pfn) 731 block_start_pfn = cc->zone->zone_start_pfn; 732 block_end_pfn = pageblock_end_pfn(pfn); 733 734 for (; pfn < end_pfn; pfn += isolated, 735 block_start_pfn = block_end_pfn, 736 block_end_pfn += pageblock_nr_pages) { 737 /* Protect pfn from changing by isolate_freepages_block */ 738 unsigned long isolate_start_pfn = pfn; 739 740 /* 741 * pfn could pass the block_end_pfn if isolated freepage 742 * is more than pageblock order. In this case, we adjust 743 * scanning range to right one. 744 */ 745 if (pfn >= block_end_pfn) { 746 block_start_pfn = pageblock_start_pfn(pfn); 747 block_end_pfn = pageblock_end_pfn(pfn); 748 } 749 750 block_end_pfn = min(block_end_pfn, end_pfn); 751 752 if (!pageblock_pfn_to_page(block_start_pfn, 753 block_end_pfn, cc->zone)) 754 break; 755 756 isolated = isolate_freepages_block(cc, &isolate_start_pfn, 757 block_end_pfn, &freelist, 0, true); 758 759 /* 760 * In strict mode, isolate_freepages_block() returns 0 if 761 * there are any holes in the block (ie. invalid PFNs or 762 * non-free pages). 763 */ 764 if (!isolated) 765 break; 766 767 /* 768 * If we managed to isolate pages, it is always (1 << n) * 769 * pageblock_nr_pages for some non-negative n. (Max order 770 * page may span two pageblocks). 771 */ 772 } 773 774 /* __isolate_free_page() does not map the pages */ 775 split_map_pages(&freelist); 776 777 if (pfn < end_pfn) { 778 /* Loop terminated early, cleanup. */ 779 release_freepages(&freelist); 780 return 0; 781 } 782 783 /* We don't use freelists for anything. */ 784 return pfn; 785 } 786 787 /* Similar to reclaim, but different enough that they don't share logic */ 788 static bool too_many_isolated(struct compact_control *cc) 789 { 790 pg_data_t *pgdat = cc->zone->zone_pgdat; 791 bool too_many; 792 793 unsigned long active, inactive, isolated; 794 795 inactive = node_page_state(pgdat, NR_INACTIVE_FILE) + 796 node_page_state(pgdat, NR_INACTIVE_ANON); 797 active = node_page_state(pgdat, NR_ACTIVE_FILE) + 798 node_page_state(pgdat, NR_ACTIVE_ANON); 799 isolated = node_page_state(pgdat, NR_ISOLATED_FILE) + 800 node_page_state(pgdat, NR_ISOLATED_ANON); 801 802 /* 803 * Allow GFP_NOFS to isolate past the limit set for regular 804 * compaction runs. This prevents an ABBA deadlock when other 805 * compactors have already isolated to the limit, but are 806 * blocked on filesystem locks held by the GFP_NOFS thread. 807 */ 808 if (cc->gfp_mask & __GFP_FS) { 809 inactive >>= 3; 810 active >>= 3; 811 } 812 813 too_many = isolated > (inactive + active) / 2; 814 if (!too_many) 815 wake_throttle_isolated(pgdat); 816 817 return too_many; 818 } 819 820 /** 821 * isolate_migratepages_block() - isolate all migrate-able pages within 822 * a single pageblock 823 * @cc: Compaction control structure. 824 * @low_pfn: The first PFN to isolate 825 * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock 826 * @mode: Isolation mode to be used. 827 * 828 * Isolate all pages that can be migrated from the range specified by 829 * [low_pfn, end_pfn). The range is expected to be within same pageblock. 830 * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion, 831 * -ENOMEM in case we could not allocate a page, or 0. 832 * cc->migrate_pfn will contain the next pfn to scan. 833 * 834 * The pages are isolated on cc->migratepages list (not required to be empty), 835 * and cc->nr_migratepages is updated accordingly. 836 */ 837 static int 838 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 839 unsigned long end_pfn, isolate_mode_t mode) 840 { 841 pg_data_t *pgdat = cc->zone->zone_pgdat; 842 unsigned long nr_scanned = 0, nr_isolated = 0; 843 struct lruvec *lruvec; 844 unsigned long flags = 0; 845 struct lruvec *locked = NULL; 846 struct folio *folio = NULL; 847 struct page *page = NULL, *valid_page = NULL; 848 struct address_space *mapping; 849 unsigned long start_pfn = low_pfn; 850 bool skip_on_failure = false; 851 unsigned long next_skip_pfn = 0; 852 bool skip_updated = false; 853 int ret = 0; 854 855 cc->migrate_pfn = low_pfn; 856 857 /* 858 * Ensure that there are not too many pages isolated from the LRU 859 * list by either parallel reclaimers or compaction. If there are, 860 * delay for some time until fewer pages are isolated 861 */ 862 while (unlikely(too_many_isolated(cc))) { 863 /* stop isolation if there are still pages not migrated */ 864 if (cc->nr_migratepages) 865 return -EAGAIN; 866 867 /* async migration should just abort */ 868 if (cc->mode == MIGRATE_ASYNC) 869 return -EAGAIN; 870 871 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 872 873 if (fatal_signal_pending(current)) 874 return -EINTR; 875 } 876 877 cond_resched(); 878 879 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { 880 skip_on_failure = true; 881 next_skip_pfn = block_end_pfn(low_pfn, cc->order); 882 } 883 884 /* Time to isolate some pages for migration */ 885 for (; low_pfn < end_pfn; low_pfn++) { 886 887 if (skip_on_failure && low_pfn >= next_skip_pfn) { 888 /* 889 * We have isolated all migration candidates in the 890 * previous order-aligned block, and did not skip it due 891 * to failure. We should migrate the pages now and 892 * hopefully succeed compaction. 893 */ 894 if (nr_isolated) 895 break; 896 897 /* 898 * We failed to isolate in the previous order-aligned 899 * block. Set the new boundary to the end of the 900 * current block. Note we can't simply increase 901 * next_skip_pfn by 1 << order, as low_pfn might have 902 * been incremented by a higher number due to skipping 903 * a compound or a high-order buddy page in the 904 * previous loop iteration. 905 */ 906 next_skip_pfn = block_end_pfn(low_pfn, cc->order); 907 } 908 909 /* 910 * Periodically drop the lock (if held) regardless of its 911 * contention, to give chance to IRQs. Abort completely if 912 * a fatal signal is pending. 913 */ 914 if (!(low_pfn % COMPACT_CLUSTER_MAX)) { 915 if (locked) { 916 unlock_page_lruvec_irqrestore(locked, flags); 917 locked = NULL; 918 } 919 920 if (fatal_signal_pending(current)) { 921 cc->contended = true; 922 ret = -EINTR; 923 924 goto fatal_pending; 925 } 926 927 cond_resched(); 928 } 929 930 nr_scanned++; 931 932 page = pfn_to_page(low_pfn); 933 934 /* 935 * Check if the pageblock has already been marked skipped. 936 * Only the first PFN is checked as the caller isolates 937 * COMPACT_CLUSTER_MAX at a time so the second call must 938 * not falsely conclude that the block should be skipped. 939 */ 940 if (!valid_page && (pageblock_aligned(low_pfn) || 941 low_pfn == cc->zone->zone_start_pfn)) { 942 if (!isolation_suitable(cc, page)) { 943 low_pfn = end_pfn; 944 folio = NULL; 945 goto isolate_abort; 946 } 947 valid_page = page; 948 } 949 950 if (PageHuge(page) && cc->alloc_contig) { 951 if (locked) { 952 unlock_page_lruvec_irqrestore(locked, flags); 953 locked = NULL; 954 } 955 956 ret = isolate_or_dissolve_huge_page(page, &cc->migratepages); 957 958 /* 959 * Fail isolation in case isolate_or_dissolve_huge_page() 960 * reports an error. In case of -ENOMEM, abort right away. 961 */ 962 if (ret < 0) { 963 /* Do not report -EBUSY down the chain */ 964 if (ret == -EBUSY) 965 ret = 0; 966 low_pfn += compound_nr(page) - 1; 967 nr_scanned += compound_nr(page) - 1; 968 goto isolate_fail; 969 } 970 971 if (PageHuge(page)) { 972 /* 973 * Hugepage was successfully isolated and placed 974 * on the cc->migratepages list. 975 */ 976 folio = page_folio(page); 977 low_pfn += folio_nr_pages(folio) - 1; 978 goto isolate_success_no_list; 979 } 980 981 /* 982 * Ok, the hugepage was dissolved. Now these pages are 983 * Buddy and cannot be re-allocated because they are 984 * isolated. Fall-through as the check below handles 985 * Buddy pages. 986 */ 987 } 988 989 /* 990 * Skip if free. We read page order here without zone lock 991 * which is generally unsafe, but the race window is small and 992 * the worst thing that can happen is that we skip some 993 * potential isolation targets. 994 */ 995 if (PageBuddy(page)) { 996 unsigned long freepage_order = buddy_order_unsafe(page); 997 998 /* 999 * Without lock, we cannot be sure that what we got is 1000 * a valid page order. Consider only values in the 1001 * valid order range to prevent low_pfn overflow. 1002 */ 1003 if (freepage_order > 0 && freepage_order <= MAX_ORDER) { 1004 low_pfn += (1UL << freepage_order) - 1; 1005 nr_scanned += (1UL << freepage_order) - 1; 1006 } 1007 continue; 1008 } 1009 1010 /* 1011 * Regardless of being on LRU, compound pages such as THP and 1012 * hugetlbfs are not to be compacted unless we are attempting 1013 * an allocation much larger than the huge page size (eg CMA). 1014 * We can potentially save a lot of iterations if we skip them 1015 * at once. The check is racy, but we can consider only valid 1016 * values and the only danger is skipping too much. 1017 */ 1018 if (PageCompound(page) && !cc->alloc_contig) { 1019 const unsigned int order = compound_order(page); 1020 1021 if (likely(order <= MAX_ORDER)) { 1022 low_pfn += (1UL << order) - 1; 1023 nr_scanned += (1UL << order) - 1; 1024 } 1025 goto isolate_fail; 1026 } 1027 1028 /* 1029 * Check may be lockless but that's ok as we recheck later. 1030 * It's possible to migrate LRU and non-lru movable pages. 1031 * Skip any other type of page 1032 */ 1033 if (!PageLRU(page)) { 1034 /* 1035 * __PageMovable can return false positive so we need 1036 * to verify it under page_lock. 1037 */ 1038 if (unlikely(__PageMovable(page)) && 1039 !PageIsolated(page)) { 1040 if (locked) { 1041 unlock_page_lruvec_irqrestore(locked, flags); 1042 locked = NULL; 1043 } 1044 1045 if (isolate_movable_page(page, mode)) { 1046 folio = page_folio(page); 1047 goto isolate_success; 1048 } 1049 } 1050 1051 goto isolate_fail; 1052 } 1053 1054 /* 1055 * Be careful not to clear PageLRU until after we're 1056 * sure the page is not being freed elsewhere -- the 1057 * page release code relies on it. 1058 */ 1059 folio = folio_get_nontail_page(page); 1060 if (unlikely(!folio)) 1061 goto isolate_fail; 1062 1063 /* 1064 * Migration will fail if an anonymous page is pinned in memory, 1065 * so avoid taking lru_lock and isolating it unnecessarily in an 1066 * admittedly racy check. 1067 */ 1068 mapping = folio_mapping(folio); 1069 if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio)) 1070 goto isolate_fail_put; 1071 1072 /* 1073 * Only allow to migrate anonymous pages in GFP_NOFS context 1074 * because those do not depend on fs locks. 1075 */ 1076 if (!(cc->gfp_mask & __GFP_FS) && mapping) 1077 goto isolate_fail_put; 1078 1079 /* Only take pages on LRU: a check now makes later tests safe */ 1080 if (!folio_test_lru(folio)) 1081 goto isolate_fail_put; 1082 1083 /* Compaction might skip unevictable pages but CMA takes them */ 1084 if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio)) 1085 goto isolate_fail_put; 1086 1087 /* 1088 * To minimise LRU disruption, the caller can indicate with 1089 * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages 1090 * it will be able to migrate without blocking - clean pages 1091 * for the most part. PageWriteback would require blocking. 1092 */ 1093 if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio)) 1094 goto isolate_fail_put; 1095 1096 if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) { 1097 bool migrate_dirty; 1098 1099 /* 1100 * Only folios without mappings or that have 1101 * a ->migrate_folio callback are possible to 1102 * migrate without blocking. However, we may 1103 * be racing with truncation, which can free 1104 * the mapping. Truncation holds the folio lock 1105 * until after the folio is removed from the page 1106 * cache so holding it ourselves is sufficient. 1107 */ 1108 if (!folio_trylock(folio)) 1109 goto isolate_fail_put; 1110 1111 mapping = folio_mapping(folio); 1112 migrate_dirty = !mapping || 1113 mapping->a_ops->migrate_folio; 1114 folio_unlock(folio); 1115 if (!migrate_dirty) 1116 goto isolate_fail_put; 1117 } 1118 1119 /* Try isolate the folio */ 1120 if (!folio_test_clear_lru(folio)) 1121 goto isolate_fail_put; 1122 1123 lruvec = folio_lruvec(folio); 1124 1125 /* If we already hold the lock, we can skip some rechecking */ 1126 if (lruvec != locked) { 1127 if (locked) 1128 unlock_page_lruvec_irqrestore(locked, flags); 1129 1130 compact_lock_irqsave(&lruvec->lru_lock, &flags, cc); 1131 locked = lruvec; 1132 1133 lruvec_memcg_debug(lruvec, folio); 1134 1135 /* 1136 * Try get exclusive access under lock. If marked for 1137 * skip, the scan is aborted unless the current context 1138 * is a rescan to reach the end of the pageblock. 1139 */ 1140 if (!skip_updated && valid_page) { 1141 skip_updated = true; 1142 if (test_and_set_skip(cc, valid_page) && 1143 !cc->finish_pageblock) { 1144 low_pfn = end_pfn; 1145 goto isolate_abort; 1146 } 1147 } 1148 1149 /* 1150 * folio become large since the non-locked check, 1151 * and it's on LRU. 1152 */ 1153 if (unlikely(folio_test_large(folio) && !cc->alloc_contig)) { 1154 low_pfn += folio_nr_pages(folio) - 1; 1155 nr_scanned += folio_nr_pages(folio) - 1; 1156 folio_set_lru(folio); 1157 goto isolate_fail_put; 1158 } 1159 } 1160 1161 /* The folio is taken off the LRU */ 1162 if (folio_test_large(folio)) 1163 low_pfn += folio_nr_pages(folio) - 1; 1164 1165 /* Successfully isolated */ 1166 lruvec_del_folio(lruvec, folio); 1167 node_stat_mod_folio(folio, 1168 NR_ISOLATED_ANON + folio_is_file_lru(folio), 1169 folio_nr_pages(folio)); 1170 1171 isolate_success: 1172 list_add(&folio->lru, &cc->migratepages); 1173 isolate_success_no_list: 1174 cc->nr_migratepages += folio_nr_pages(folio); 1175 nr_isolated += folio_nr_pages(folio); 1176 nr_scanned += folio_nr_pages(folio) - 1; 1177 1178 /* 1179 * Avoid isolating too much unless this block is being 1180 * fully scanned (e.g. dirty/writeback pages, parallel allocation) 1181 * or a lock is contended. For contention, isolate quickly to 1182 * potentially remove one source of contention. 1183 */ 1184 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX && 1185 !cc->finish_pageblock && !cc->contended) { 1186 ++low_pfn; 1187 break; 1188 } 1189 1190 continue; 1191 1192 isolate_fail_put: 1193 /* Avoid potential deadlock in freeing page under lru_lock */ 1194 if (locked) { 1195 unlock_page_lruvec_irqrestore(locked, flags); 1196 locked = NULL; 1197 } 1198 folio_put(folio); 1199 1200 isolate_fail: 1201 if (!skip_on_failure && ret != -ENOMEM) 1202 continue; 1203 1204 /* 1205 * We have isolated some pages, but then failed. Release them 1206 * instead of migrating, as we cannot form the cc->order buddy 1207 * page anyway. 1208 */ 1209 if (nr_isolated) { 1210 if (locked) { 1211 unlock_page_lruvec_irqrestore(locked, flags); 1212 locked = NULL; 1213 } 1214 putback_movable_pages(&cc->migratepages); 1215 cc->nr_migratepages = 0; 1216 nr_isolated = 0; 1217 } 1218 1219 if (low_pfn < next_skip_pfn) { 1220 low_pfn = next_skip_pfn - 1; 1221 /* 1222 * The check near the loop beginning would have updated 1223 * next_skip_pfn too, but this is a bit simpler. 1224 */ 1225 next_skip_pfn += 1UL << cc->order; 1226 } 1227 1228 if (ret == -ENOMEM) 1229 break; 1230 } 1231 1232 /* 1233 * The PageBuddy() check could have potentially brought us outside 1234 * the range to be scanned. 1235 */ 1236 if (unlikely(low_pfn > end_pfn)) 1237 low_pfn = end_pfn; 1238 1239 folio = NULL; 1240 1241 isolate_abort: 1242 if (locked) 1243 unlock_page_lruvec_irqrestore(locked, flags); 1244 if (folio) { 1245 folio_set_lru(folio); 1246 folio_put(folio); 1247 } 1248 1249 /* 1250 * Update the cached scanner pfn once the pageblock has been scanned. 1251 * Pages will either be migrated in which case there is no point 1252 * scanning in the near future or migration failed in which case the 1253 * failure reason may persist. The block is marked for skipping if 1254 * there were no pages isolated in the block or if the block is 1255 * rescanned twice in a row. 1256 */ 1257 if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) { 1258 if (!cc->no_set_skip_hint && valid_page && !skip_updated) 1259 set_pageblock_skip(valid_page); 1260 update_cached_migrate(cc, low_pfn); 1261 } 1262 1263 trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn, 1264 nr_scanned, nr_isolated); 1265 1266 fatal_pending: 1267 cc->total_migrate_scanned += nr_scanned; 1268 if (nr_isolated) 1269 count_compact_events(COMPACTISOLATED, nr_isolated); 1270 1271 cc->migrate_pfn = low_pfn; 1272 1273 return ret; 1274 } 1275 1276 /** 1277 * isolate_migratepages_range() - isolate migrate-able pages in a PFN range 1278 * @cc: Compaction control structure. 1279 * @start_pfn: The first PFN to start isolating. 1280 * @end_pfn: The one-past-last PFN. 1281 * 1282 * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM 1283 * in case we could not allocate a page, or 0. 1284 */ 1285 int 1286 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, 1287 unsigned long end_pfn) 1288 { 1289 unsigned long pfn, block_start_pfn, block_end_pfn; 1290 int ret = 0; 1291 1292 /* Scan block by block. First and last block may be incomplete */ 1293 pfn = start_pfn; 1294 block_start_pfn = pageblock_start_pfn(pfn); 1295 if (block_start_pfn < cc->zone->zone_start_pfn) 1296 block_start_pfn = cc->zone->zone_start_pfn; 1297 block_end_pfn = pageblock_end_pfn(pfn); 1298 1299 for (; pfn < end_pfn; pfn = block_end_pfn, 1300 block_start_pfn = block_end_pfn, 1301 block_end_pfn += pageblock_nr_pages) { 1302 1303 block_end_pfn = min(block_end_pfn, end_pfn); 1304 1305 if (!pageblock_pfn_to_page(block_start_pfn, 1306 block_end_pfn, cc->zone)) 1307 continue; 1308 1309 ret = isolate_migratepages_block(cc, pfn, block_end_pfn, 1310 ISOLATE_UNEVICTABLE); 1311 1312 if (ret) 1313 break; 1314 1315 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX) 1316 break; 1317 } 1318 1319 return ret; 1320 } 1321 1322 #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 1323 #ifdef CONFIG_COMPACTION 1324 1325 static bool suitable_migration_source(struct compact_control *cc, 1326 struct page *page) 1327 { 1328 int block_mt; 1329 1330 if (pageblock_skip_persistent(page)) 1331 return false; 1332 1333 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) 1334 return true; 1335 1336 block_mt = get_pageblock_migratetype(page); 1337 1338 if (cc->migratetype == MIGRATE_MOVABLE) 1339 return is_migrate_movable(block_mt); 1340 else 1341 return block_mt == cc->migratetype; 1342 } 1343 1344 /* Returns true if the page is within a block suitable for migration to */ 1345 static bool suitable_migration_target(struct compact_control *cc, 1346 struct page *page) 1347 { 1348 /* If the page is a large free page, then disallow migration */ 1349 if (PageBuddy(page)) { 1350 /* 1351 * We are checking page_order without zone->lock taken. But 1352 * the only small danger is that we skip a potentially suitable 1353 * pageblock, so it's not worth to check order for valid range. 1354 */ 1355 if (buddy_order_unsafe(page) >= pageblock_order) 1356 return false; 1357 } 1358 1359 if (cc->ignore_block_suitable) 1360 return true; 1361 1362 /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ 1363 if (is_migrate_movable(get_pageblock_migratetype(page))) 1364 return true; 1365 1366 /* Otherwise skip the block */ 1367 return false; 1368 } 1369 1370 static inline unsigned int 1371 freelist_scan_limit(struct compact_control *cc) 1372 { 1373 unsigned short shift = BITS_PER_LONG - 1; 1374 1375 return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1; 1376 } 1377 1378 /* 1379 * Test whether the free scanner has reached the same or lower pageblock than 1380 * the migration scanner, and compaction should thus terminate. 1381 */ 1382 static inline bool compact_scanners_met(struct compact_control *cc) 1383 { 1384 return (cc->free_pfn >> pageblock_order) 1385 <= (cc->migrate_pfn >> pageblock_order); 1386 } 1387 1388 /* 1389 * Used when scanning for a suitable migration target which scans freelists 1390 * in reverse. Reorders the list such as the unscanned pages are scanned 1391 * first on the next iteration of the free scanner 1392 */ 1393 static void 1394 move_freelist_head(struct list_head *freelist, struct page *freepage) 1395 { 1396 LIST_HEAD(sublist); 1397 1398 if (!list_is_last(freelist, &freepage->lru)) { 1399 list_cut_before(&sublist, freelist, &freepage->lru); 1400 list_splice_tail(&sublist, freelist); 1401 } 1402 } 1403 1404 /* 1405 * Similar to move_freelist_head except used by the migration scanner 1406 * when scanning forward. It's possible for these list operations to 1407 * move against each other if they search the free list exactly in 1408 * lockstep. 1409 */ 1410 static void 1411 move_freelist_tail(struct list_head *freelist, struct page *freepage) 1412 { 1413 LIST_HEAD(sublist); 1414 1415 if (!list_is_first(freelist, &freepage->lru)) { 1416 list_cut_position(&sublist, freelist, &freepage->lru); 1417 list_splice_tail(&sublist, freelist); 1418 } 1419 } 1420 1421 static void 1422 fast_isolate_around(struct compact_control *cc, unsigned long pfn) 1423 { 1424 unsigned long start_pfn, end_pfn; 1425 struct page *page; 1426 1427 /* Do not search around if there are enough pages already */ 1428 if (cc->nr_freepages >= cc->nr_migratepages) 1429 return; 1430 1431 /* Minimise scanning during async compaction */ 1432 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC) 1433 return; 1434 1435 /* Pageblock boundaries */ 1436 start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn); 1437 end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)); 1438 1439 page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone); 1440 if (!page) 1441 return; 1442 1443 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false); 1444 1445 /* Skip this pageblock in the future as it's full or nearly full */ 1446 if (start_pfn == end_pfn && !cc->no_set_skip_hint) 1447 set_pageblock_skip(page); 1448 } 1449 1450 /* Search orders in round-robin fashion */ 1451 static int next_search_order(struct compact_control *cc, int order) 1452 { 1453 order--; 1454 if (order < 0) 1455 order = cc->order - 1; 1456 1457 /* Search wrapped around? */ 1458 if (order == cc->search_order) { 1459 cc->search_order--; 1460 if (cc->search_order < 0) 1461 cc->search_order = cc->order - 1; 1462 return -1; 1463 } 1464 1465 return order; 1466 } 1467 1468 static void fast_isolate_freepages(struct compact_control *cc) 1469 { 1470 unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1); 1471 unsigned int nr_scanned = 0, total_isolated = 0; 1472 unsigned long low_pfn, min_pfn, highest = 0; 1473 unsigned long nr_isolated = 0; 1474 unsigned long distance; 1475 struct page *page = NULL; 1476 bool scan_start = false; 1477 int order; 1478 1479 /* Full compaction passes in a negative order */ 1480 if (cc->order <= 0) 1481 return; 1482 1483 /* 1484 * If starting the scan, use a deeper search and use the highest 1485 * PFN found if a suitable one is not found. 1486 */ 1487 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) { 1488 limit = pageblock_nr_pages >> 1; 1489 scan_start = true; 1490 } 1491 1492 /* 1493 * Preferred point is in the top quarter of the scan space but take 1494 * a pfn from the top half if the search is problematic. 1495 */ 1496 distance = (cc->free_pfn - cc->migrate_pfn); 1497 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2)); 1498 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1)); 1499 1500 if (WARN_ON_ONCE(min_pfn > low_pfn)) 1501 low_pfn = min_pfn; 1502 1503 /* 1504 * Search starts from the last successful isolation order or the next 1505 * order to search after a previous failure 1506 */ 1507 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order); 1508 1509 for (order = cc->search_order; 1510 !page && order >= 0; 1511 order = next_search_order(cc, order)) { 1512 struct free_area *area = &cc->zone->free_area[order]; 1513 struct list_head *freelist; 1514 struct page *freepage; 1515 unsigned long flags; 1516 unsigned int order_scanned = 0; 1517 unsigned long high_pfn = 0; 1518 1519 if (!area->nr_free) 1520 continue; 1521 1522 spin_lock_irqsave(&cc->zone->lock, flags); 1523 freelist = &area->free_list[MIGRATE_MOVABLE]; 1524 list_for_each_entry_reverse(freepage, freelist, buddy_list) { 1525 unsigned long pfn; 1526 1527 order_scanned++; 1528 nr_scanned++; 1529 pfn = page_to_pfn(freepage); 1530 1531 if (pfn >= highest) 1532 highest = max(pageblock_start_pfn(pfn), 1533 cc->zone->zone_start_pfn); 1534 1535 if (pfn >= low_pfn) { 1536 cc->fast_search_fail = 0; 1537 cc->search_order = order; 1538 page = freepage; 1539 break; 1540 } 1541 1542 if (pfn >= min_pfn && pfn > high_pfn) { 1543 high_pfn = pfn; 1544 1545 /* Shorten the scan if a candidate is found */ 1546 limit >>= 1; 1547 } 1548 1549 if (order_scanned >= limit) 1550 break; 1551 } 1552 1553 /* Use a maximum candidate pfn if a preferred one was not found */ 1554 if (!page && high_pfn) { 1555 page = pfn_to_page(high_pfn); 1556 1557 /* Update freepage for the list reorder below */ 1558 freepage = page; 1559 } 1560 1561 /* Reorder to so a future search skips recent pages */ 1562 move_freelist_head(freelist, freepage); 1563 1564 /* Isolate the page if available */ 1565 if (page) { 1566 if (__isolate_free_page(page, order)) { 1567 set_page_private(page, order); 1568 nr_isolated = 1 << order; 1569 nr_scanned += nr_isolated - 1; 1570 total_isolated += nr_isolated; 1571 cc->nr_freepages += nr_isolated; 1572 list_add_tail(&page->lru, &cc->freepages); 1573 count_compact_events(COMPACTISOLATED, nr_isolated); 1574 } else { 1575 /* If isolation fails, abort the search */ 1576 order = cc->search_order + 1; 1577 page = NULL; 1578 } 1579 } 1580 1581 spin_unlock_irqrestore(&cc->zone->lock, flags); 1582 1583 /* Skip fast search if enough freepages isolated */ 1584 if (cc->nr_freepages >= cc->nr_migratepages) 1585 break; 1586 1587 /* 1588 * Smaller scan on next order so the total scan is related 1589 * to freelist_scan_limit. 1590 */ 1591 if (order_scanned >= limit) 1592 limit = max(1U, limit >> 1); 1593 } 1594 1595 trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn, 1596 nr_scanned, total_isolated); 1597 1598 if (!page) { 1599 cc->fast_search_fail++; 1600 if (scan_start) { 1601 /* 1602 * Use the highest PFN found above min. If one was 1603 * not found, be pessimistic for direct compaction 1604 * and use the min mark. 1605 */ 1606 if (highest >= min_pfn) { 1607 page = pfn_to_page(highest); 1608 cc->free_pfn = highest; 1609 } else { 1610 if (cc->direct_compaction && pfn_valid(min_pfn)) { 1611 page = pageblock_pfn_to_page(min_pfn, 1612 min(pageblock_end_pfn(min_pfn), 1613 zone_end_pfn(cc->zone)), 1614 cc->zone); 1615 cc->free_pfn = min_pfn; 1616 } 1617 } 1618 } 1619 } 1620 1621 if (highest && highest >= cc->zone->compact_cached_free_pfn) { 1622 highest -= pageblock_nr_pages; 1623 cc->zone->compact_cached_free_pfn = highest; 1624 } 1625 1626 cc->total_free_scanned += nr_scanned; 1627 if (!page) 1628 return; 1629 1630 low_pfn = page_to_pfn(page); 1631 fast_isolate_around(cc, low_pfn); 1632 } 1633 1634 /* 1635 * Based on information in the current compact_control, find blocks 1636 * suitable for isolating free pages from and then isolate them. 1637 */ 1638 static void isolate_freepages(struct compact_control *cc) 1639 { 1640 struct zone *zone = cc->zone; 1641 struct page *page; 1642 unsigned long block_start_pfn; /* start of current pageblock */ 1643 unsigned long isolate_start_pfn; /* exact pfn we start at */ 1644 unsigned long block_end_pfn; /* end of current pageblock */ 1645 unsigned long low_pfn; /* lowest pfn scanner is able to scan */ 1646 struct list_head *freelist = &cc->freepages; 1647 unsigned int stride; 1648 1649 /* Try a small search of the free lists for a candidate */ 1650 fast_isolate_freepages(cc); 1651 if (cc->nr_freepages) 1652 goto splitmap; 1653 1654 /* 1655 * Initialise the free scanner. The starting point is where we last 1656 * successfully isolated from, zone-cached value, or the end of the 1657 * zone when isolating for the first time. For looping we also need 1658 * this pfn aligned down to the pageblock boundary, because we do 1659 * block_start_pfn -= pageblock_nr_pages in the for loop. 1660 * For ending point, take care when isolating in last pageblock of a 1661 * zone which ends in the middle of a pageblock. 1662 * The low boundary is the end of the pageblock the migration scanner 1663 * is using. 1664 */ 1665 isolate_start_pfn = cc->free_pfn; 1666 block_start_pfn = pageblock_start_pfn(isolate_start_pfn); 1667 block_end_pfn = min(block_start_pfn + pageblock_nr_pages, 1668 zone_end_pfn(zone)); 1669 low_pfn = pageblock_end_pfn(cc->migrate_pfn); 1670 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1; 1671 1672 /* 1673 * Isolate free pages until enough are available to migrate the 1674 * pages on cc->migratepages. We stop searching if the migrate 1675 * and free page scanners meet or enough free pages are isolated. 1676 */ 1677 for (; block_start_pfn >= low_pfn; 1678 block_end_pfn = block_start_pfn, 1679 block_start_pfn -= pageblock_nr_pages, 1680 isolate_start_pfn = block_start_pfn) { 1681 unsigned long nr_isolated; 1682 1683 /* 1684 * This can iterate a massively long zone without finding any 1685 * suitable migration targets, so periodically check resched. 1686 */ 1687 if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages))) 1688 cond_resched(); 1689 1690 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn, 1691 zone); 1692 if (!page) { 1693 unsigned long next_pfn; 1694 1695 next_pfn = skip_offline_sections_reverse(block_start_pfn); 1696 if (next_pfn) 1697 block_start_pfn = max(next_pfn, low_pfn); 1698 1699 continue; 1700 } 1701 1702 /* Check the block is suitable for migration */ 1703 if (!suitable_migration_target(cc, page)) 1704 continue; 1705 1706 /* If isolation recently failed, do not retry */ 1707 if (!isolation_suitable(cc, page)) 1708 continue; 1709 1710 /* Found a block suitable for isolating free pages from. */ 1711 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn, 1712 block_end_pfn, freelist, stride, false); 1713 1714 /* Update the skip hint if the full pageblock was scanned */ 1715 if (isolate_start_pfn == block_end_pfn) 1716 update_pageblock_skip(cc, page, block_start_pfn - 1717 pageblock_nr_pages); 1718 1719 /* Are enough freepages isolated? */ 1720 if (cc->nr_freepages >= cc->nr_migratepages) { 1721 if (isolate_start_pfn >= block_end_pfn) { 1722 /* 1723 * Restart at previous pageblock if more 1724 * freepages can be isolated next time. 1725 */ 1726 isolate_start_pfn = 1727 block_start_pfn - pageblock_nr_pages; 1728 } 1729 break; 1730 } else if (isolate_start_pfn < block_end_pfn) { 1731 /* 1732 * If isolation failed early, do not continue 1733 * needlessly. 1734 */ 1735 break; 1736 } 1737 1738 /* Adjust stride depending on isolation */ 1739 if (nr_isolated) { 1740 stride = 1; 1741 continue; 1742 } 1743 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1); 1744 } 1745 1746 /* 1747 * Record where the free scanner will restart next time. Either we 1748 * broke from the loop and set isolate_start_pfn based on the last 1749 * call to isolate_freepages_block(), or we met the migration scanner 1750 * and the loop terminated due to isolate_start_pfn < low_pfn 1751 */ 1752 cc->free_pfn = isolate_start_pfn; 1753 1754 splitmap: 1755 /* __isolate_free_page() does not map the pages */ 1756 split_map_pages(freelist); 1757 } 1758 1759 /* 1760 * This is a migrate-callback that "allocates" freepages by taking pages 1761 * from the isolated freelists in the block we are migrating to. 1762 */ 1763 static struct folio *compaction_alloc(struct folio *src, unsigned long data) 1764 { 1765 struct compact_control *cc = (struct compact_control *)data; 1766 struct folio *dst; 1767 1768 if (list_empty(&cc->freepages)) { 1769 isolate_freepages(cc); 1770 1771 if (list_empty(&cc->freepages)) 1772 return NULL; 1773 } 1774 1775 dst = list_entry(cc->freepages.next, struct folio, lru); 1776 list_del(&dst->lru); 1777 cc->nr_freepages--; 1778 1779 return dst; 1780 } 1781 1782 /* 1783 * This is a migrate-callback that "frees" freepages back to the isolated 1784 * freelist. All pages on the freelist are from the same zone, so there is no 1785 * special handling needed for NUMA. 1786 */ 1787 static void compaction_free(struct folio *dst, unsigned long data) 1788 { 1789 struct compact_control *cc = (struct compact_control *)data; 1790 1791 list_add(&dst->lru, &cc->freepages); 1792 cc->nr_freepages++; 1793 } 1794 1795 /* possible outcome of isolate_migratepages */ 1796 typedef enum { 1797 ISOLATE_ABORT, /* Abort compaction now */ 1798 ISOLATE_NONE, /* No pages isolated, continue scanning */ 1799 ISOLATE_SUCCESS, /* Pages isolated, migrate */ 1800 } isolate_migrate_t; 1801 1802 /* 1803 * Allow userspace to control policy on scanning the unevictable LRU for 1804 * compactable pages. 1805 */ 1806 static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT; 1807 /* 1808 * Tunable for proactive compaction. It determines how 1809 * aggressively the kernel should compact memory in the 1810 * background. It takes values in the range [0, 100]. 1811 */ 1812 static unsigned int __read_mostly sysctl_compaction_proactiveness = 20; 1813 static int sysctl_extfrag_threshold = 500; 1814 static int __read_mostly sysctl_compact_memory; 1815 1816 static inline void 1817 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) 1818 { 1819 if (cc->fast_start_pfn == ULONG_MAX) 1820 return; 1821 1822 if (!cc->fast_start_pfn) 1823 cc->fast_start_pfn = pfn; 1824 1825 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn); 1826 } 1827 1828 static inline unsigned long 1829 reinit_migrate_pfn(struct compact_control *cc) 1830 { 1831 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX) 1832 return cc->migrate_pfn; 1833 1834 cc->migrate_pfn = cc->fast_start_pfn; 1835 cc->fast_start_pfn = ULONG_MAX; 1836 1837 return cc->migrate_pfn; 1838 } 1839 1840 /* 1841 * Briefly search the free lists for a migration source that already has 1842 * some free pages to reduce the number of pages that need migration 1843 * before a pageblock is free. 1844 */ 1845 static unsigned long fast_find_migrateblock(struct compact_control *cc) 1846 { 1847 unsigned int limit = freelist_scan_limit(cc); 1848 unsigned int nr_scanned = 0; 1849 unsigned long distance; 1850 unsigned long pfn = cc->migrate_pfn; 1851 unsigned long high_pfn; 1852 int order; 1853 bool found_block = false; 1854 1855 /* Skip hints are relied on to avoid repeats on the fast search */ 1856 if (cc->ignore_skip_hint) 1857 return pfn; 1858 1859 /* 1860 * If the pageblock should be finished then do not select a different 1861 * pageblock. 1862 */ 1863 if (cc->finish_pageblock) 1864 return pfn; 1865 1866 /* 1867 * If the migrate_pfn is not at the start of a zone or the start 1868 * of a pageblock then assume this is a continuation of a previous 1869 * scan restarted due to COMPACT_CLUSTER_MAX. 1870 */ 1871 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn)) 1872 return pfn; 1873 1874 /* 1875 * For smaller orders, just linearly scan as the number of pages 1876 * to migrate should be relatively small and does not necessarily 1877 * justify freeing up a large block for a small allocation. 1878 */ 1879 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER) 1880 return pfn; 1881 1882 /* 1883 * Only allow kcompactd and direct requests for movable pages to 1884 * quickly clear out a MOVABLE pageblock for allocation. This 1885 * reduces the risk that a large movable pageblock is freed for 1886 * an unmovable/reclaimable small allocation. 1887 */ 1888 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE) 1889 return pfn; 1890 1891 /* 1892 * When starting the migration scanner, pick any pageblock within the 1893 * first half of the search space. Otherwise try and pick a pageblock 1894 * within the first eighth to reduce the chances that a migration 1895 * target later becomes a source. 1896 */ 1897 distance = (cc->free_pfn - cc->migrate_pfn) >> 1; 1898 if (cc->migrate_pfn != cc->zone->zone_start_pfn) 1899 distance >>= 2; 1900 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance); 1901 1902 for (order = cc->order - 1; 1903 order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit; 1904 order--) { 1905 struct free_area *area = &cc->zone->free_area[order]; 1906 struct list_head *freelist; 1907 unsigned long flags; 1908 struct page *freepage; 1909 1910 if (!area->nr_free) 1911 continue; 1912 1913 spin_lock_irqsave(&cc->zone->lock, flags); 1914 freelist = &area->free_list[MIGRATE_MOVABLE]; 1915 list_for_each_entry(freepage, freelist, buddy_list) { 1916 unsigned long free_pfn; 1917 1918 if (nr_scanned++ >= limit) { 1919 move_freelist_tail(freelist, freepage); 1920 break; 1921 } 1922 1923 free_pfn = page_to_pfn(freepage); 1924 if (free_pfn < high_pfn) { 1925 /* 1926 * Avoid if skipped recently. Ideally it would 1927 * move to the tail but even safe iteration of 1928 * the list assumes an entry is deleted, not 1929 * reordered. 1930 */ 1931 if (get_pageblock_skip(freepage)) 1932 continue; 1933 1934 /* Reorder to so a future search skips recent pages */ 1935 move_freelist_tail(freelist, freepage); 1936 1937 update_fast_start_pfn(cc, free_pfn); 1938 pfn = pageblock_start_pfn(free_pfn); 1939 if (pfn < cc->zone->zone_start_pfn) 1940 pfn = cc->zone->zone_start_pfn; 1941 cc->fast_search_fail = 0; 1942 found_block = true; 1943 break; 1944 } 1945 } 1946 spin_unlock_irqrestore(&cc->zone->lock, flags); 1947 } 1948 1949 cc->total_migrate_scanned += nr_scanned; 1950 1951 /* 1952 * If fast scanning failed then use a cached entry for a page block 1953 * that had free pages as the basis for starting a linear scan. 1954 */ 1955 if (!found_block) { 1956 cc->fast_search_fail++; 1957 pfn = reinit_migrate_pfn(cc); 1958 } 1959 return pfn; 1960 } 1961 1962 /* 1963 * Isolate all pages that can be migrated from the first suitable block, 1964 * starting at the block pointed to by the migrate scanner pfn within 1965 * compact_control. 1966 */ 1967 static isolate_migrate_t isolate_migratepages(struct compact_control *cc) 1968 { 1969 unsigned long block_start_pfn; 1970 unsigned long block_end_pfn; 1971 unsigned long low_pfn; 1972 struct page *page; 1973 const isolate_mode_t isolate_mode = 1974 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | 1975 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0); 1976 bool fast_find_block; 1977 1978 /* 1979 * Start at where we last stopped, or beginning of the zone as 1980 * initialized by compact_zone(). The first failure will use 1981 * the lowest PFN as the starting point for linear scanning. 1982 */ 1983 low_pfn = fast_find_migrateblock(cc); 1984 block_start_pfn = pageblock_start_pfn(low_pfn); 1985 if (block_start_pfn < cc->zone->zone_start_pfn) 1986 block_start_pfn = cc->zone->zone_start_pfn; 1987 1988 /* 1989 * fast_find_migrateblock() has already ensured the pageblock is not 1990 * set with a skipped flag, so to avoid the isolation_suitable check 1991 * below again, check whether the fast search was successful. 1992 */ 1993 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail; 1994 1995 /* Only scan within a pageblock boundary */ 1996 block_end_pfn = pageblock_end_pfn(low_pfn); 1997 1998 /* 1999 * Iterate over whole pageblocks until we find the first suitable. 2000 * Do not cross the free scanner. 2001 */ 2002 for (; block_end_pfn <= cc->free_pfn; 2003 fast_find_block = false, 2004 cc->migrate_pfn = low_pfn = block_end_pfn, 2005 block_start_pfn = block_end_pfn, 2006 block_end_pfn += pageblock_nr_pages) { 2007 2008 /* 2009 * This can potentially iterate a massively long zone with 2010 * many pageblocks unsuitable, so periodically check if we 2011 * need to schedule. 2012 */ 2013 if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages))) 2014 cond_resched(); 2015 2016 page = pageblock_pfn_to_page(block_start_pfn, 2017 block_end_pfn, cc->zone); 2018 if (!page) { 2019 unsigned long next_pfn; 2020 2021 next_pfn = skip_offline_sections(block_start_pfn); 2022 if (next_pfn) 2023 block_end_pfn = min(next_pfn, cc->free_pfn); 2024 continue; 2025 } 2026 2027 /* 2028 * If isolation recently failed, do not retry. Only check the 2029 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock 2030 * to be visited multiple times. Assume skip was checked 2031 * before making it "skip" so other compaction instances do 2032 * not scan the same block. 2033 */ 2034 if ((pageblock_aligned(low_pfn) || 2035 low_pfn == cc->zone->zone_start_pfn) && 2036 !fast_find_block && !isolation_suitable(cc, page)) 2037 continue; 2038 2039 /* 2040 * For async direct compaction, only scan the pageblocks of the 2041 * same migratetype without huge pages. Async direct compaction 2042 * is optimistic to see if the minimum amount of work satisfies 2043 * the allocation. The cached PFN is updated as it's possible 2044 * that all remaining blocks between source and target are 2045 * unsuitable and the compaction scanners fail to meet. 2046 */ 2047 if (!suitable_migration_source(cc, page)) { 2048 update_cached_migrate(cc, block_end_pfn); 2049 continue; 2050 } 2051 2052 /* Perform the isolation */ 2053 if (isolate_migratepages_block(cc, low_pfn, block_end_pfn, 2054 isolate_mode)) 2055 return ISOLATE_ABORT; 2056 2057 /* 2058 * Either we isolated something and proceed with migration. Or 2059 * we failed and compact_zone should decide if we should 2060 * continue or not. 2061 */ 2062 break; 2063 } 2064 2065 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE; 2066 } 2067 2068 /* 2069 * order == -1 is expected when compacting via 2070 * /proc/sys/vm/compact_memory 2071 */ 2072 static inline bool is_via_compact_memory(int order) 2073 { 2074 return order == -1; 2075 } 2076 2077 /* 2078 * Determine whether kswapd is (or recently was!) running on this node. 2079 * 2080 * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't 2081 * zero it. 2082 */ 2083 static bool kswapd_is_running(pg_data_t *pgdat) 2084 { 2085 bool running; 2086 2087 pgdat_kswapd_lock(pgdat); 2088 running = pgdat->kswapd && task_is_running(pgdat->kswapd); 2089 pgdat_kswapd_unlock(pgdat); 2090 2091 return running; 2092 } 2093 2094 /* 2095 * A zone's fragmentation score is the external fragmentation wrt to the 2096 * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100]. 2097 */ 2098 static unsigned int fragmentation_score_zone(struct zone *zone) 2099 { 2100 return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER); 2101 } 2102 2103 /* 2104 * A weighted zone's fragmentation score is the external fragmentation 2105 * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It 2106 * returns a value in the range [0, 100]. 2107 * 2108 * The scaling factor ensures that proactive compaction focuses on larger 2109 * zones like ZONE_NORMAL, rather than smaller, specialized zones like 2110 * ZONE_DMA32. For smaller zones, the score value remains close to zero, 2111 * and thus never exceeds the high threshold for proactive compaction. 2112 */ 2113 static unsigned int fragmentation_score_zone_weighted(struct zone *zone) 2114 { 2115 unsigned long score; 2116 2117 score = zone->present_pages * fragmentation_score_zone(zone); 2118 return div64_ul(score, zone->zone_pgdat->node_present_pages + 1); 2119 } 2120 2121 /* 2122 * The per-node proactive (background) compaction process is started by its 2123 * corresponding kcompactd thread when the node's fragmentation score 2124 * exceeds the high threshold. The compaction process remains active till 2125 * the node's score falls below the low threshold, or one of the back-off 2126 * conditions is met. 2127 */ 2128 static unsigned int fragmentation_score_node(pg_data_t *pgdat) 2129 { 2130 unsigned int score = 0; 2131 int zoneid; 2132 2133 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 2134 struct zone *zone; 2135 2136 zone = &pgdat->node_zones[zoneid]; 2137 if (!populated_zone(zone)) 2138 continue; 2139 score += fragmentation_score_zone_weighted(zone); 2140 } 2141 2142 return score; 2143 } 2144 2145 static unsigned int fragmentation_score_wmark(bool low) 2146 { 2147 unsigned int wmark_low; 2148 2149 /* 2150 * Cap the low watermark to avoid excessive compaction 2151 * activity in case a user sets the proactiveness tunable 2152 * close to 100 (maximum). 2153 */ 2154 wmark_low = max(100U - sysctl_compaction_proactiveness, 5U); 2155 return low ? wmark_low : min(wmark_low + 10, 100U); 2156 } 2157 2158 static bool should_proactive_compact_node(pg_data_t *pgdat) 2159 { 2160 int wmark_high; 2161 2162 if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat)) 2163 return false; 2164 2165 wmark_high = fragmentation_score_wmark(false); 2166 return fragmentation_score_node(pgdat) > wmark_high; 2167 } 2168 2169 static enum compact_result __compact_finished(struct compact_control *cc) 2170 { 2171 unsigned int order; 2172 const int migratetype = cc->migratetype; 2173 int ret; 2174 2175 /* Compaction run completes if the migrate and free scanner meet */ 2176 if (compact_scanners_met(cc)) { 2177 /* Let the next compaction start anew. */ 2178 reset_cached_positions(cc->zone); 2179 2180 /* 2181 * Mark that the PG_migrate_skip information should be cleared 2182 * by kswapd when it goes to sleep. kcompactd does not set the 2183 * flag itself as the decision to be clear should be directly 2184 * based on an allocation request. 2185 */ 2186 if (cc->direct_compaction) 2187 cc->zone->compact_blockskip_flush = true; 2188 2189 if (cc->whole_zone) 2190 return COMPACT_COMPLETE; 2191 else 2192 return COMPACT_PARTIAL_SKIPPED; 2193 } 2194 2195 if (cc->proactive_compaction) { 2196 int score, wmark_low; 2197 pg_data_t *pgdat; 2198 2199 pgdat = cc->zone->zone_pgdat; 2200 if (kswapd_is_running(pgdat)) 2201 return COMPACT_PARTIAL_SKIPPED; 2202 2203 score = fragmentation_score_zone(cc->zone); 2204 wmark_low = fragmentation_score_wmark(true); 2205 2206 if (score > wmark_low) 2207 ret = COMPACT_CONTINUE; 2208 else 2209 ret = COMPACT_SUCCESS; 2210 2211 goto out; 2212 } 2213 2214 if (is_via_compact_memory(cc->order)) 2215 return COMPACT_CONTINUE; 2216 2217 /* 2218 * Always finish scanning a pageblock to reduce the possibility of 2219 * fallbacks in the future. This is particularly important when 2220 * migration source is unmovable/reclaimable but it's not worth 2221 * special casing. 2222 */ 2223 if (!pageblock_aligned(cc->migrate_pfn)) 2224 return COMPACT_CONTINUE; 2225 2226 /* Direct compactor: Is a suitable page free? */ 2227 ret = COMPACT_NO_SUITABLE_PAGE; 2228 for (order = cc->order; order <= MAX_ORDER; order++) { 2229 struct free_area *area = &cc->zone->free_area[order]; 2230 bool can_steal; 2231 2232 /* Job done if page is free of the right migratetype */ 2233 if (!free_area_empty(area, migratetype)) 2234 return COMPACT_SUCCESS; 2235 2236 #ifdef CONFIG_CMA 2237 /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */ 2238 if (migratetype == MIGRATE_MOVABLE && 2239 !free_area_empty(area, MIGRATE_CMA)) 2240 return COMPACT_SUCCESS; 2241 #endif 2242 /* 2243 * Job done if allocation would steal freepages from 2244 * other migratetype buddy lists. 2245 */ 2246 if (find_suitable_fallback(area, order, migratetype, 2247 true, &can_steal) != -1) 2248 /* 2249 * Movable pages are OK in any pageblock. If we are 2250 * stealing for a non-movable allocation, make sure 2251 * we finish compacting the current pageblock first 2252 * (which is assured by the above migrate_pfn align 2253 * check) so it is as free as possible and we won't 2254 * have to steal another one soon. 2255 */ 2256 return COMPACT_SUCCESS; 2257 } 2258 2259 out: 2260 if (cc->contended || fatal_signal_pending(current)) 2261 ret = COMPACT_CONTENDED; 2262 2263 return ret; 2264 } 2265 2266 static enum compact_result compact_finished(struct compact_control *cc) 2267 { 2268 int ret; 2269 2270 ret = __compact_finished(cc); 2271 trace_mm_compaction_finished(cc->zone, cc->order, ret); 2272 if (ret == COMPACT_NO_SUITABLE_PAGE) 2273 ret = COMPACT_CONTINUE; 2274 2275 return ret; 2276 } 2277 2278 static bool __compaction_suitable(struct zone *zone, int order, 2279 int highest_zoneidx, 2280 unsigned long wmark_target) 2281 { 2282 unsigned long watermark; 2283 /* 2284 * Watermarks for order-0 must be met for compaction to be able to 2285 * isolate free pages for migration targets. This means that the 2286 * watermark and alloc_flags have to match, or be more pessimistic than 2287 * the check in __isolate_free_page(). We don't use the direct 2288 * compactor's alloc_flags, as they are not relevant for freepage 2289 * isolation. We however do use the direct compactor's highest_zoneidx 2290 * to skip over zones where lowmem reserves would prevent allocation 2291 * even if compaction succeeds. 2292 * For costly orders, we require low watermark instead of min for 2293 * compaction to proceed to increase its chances. 2294 * ALLOC_CMA is used, as pages in CMA pageblocks are considered 2295 * suitable migration targets 2296 */ 2297 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ? 2298 low_wmark_pages(zone) : min_wmark_pages(zone); 2299 watermark += compact_gap(order); 2300 return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx, 2301 ALLOC_CMA, wmark_target); 2302 } 2303 2304 /* 2305 * compaction_suitable: Is this suitable to run compaction on this zone now? 2306 */ 2307 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx) 2308 { 2309 enum compact_result compact_result; 2310 bool suitable; 2311 2312 suitable = __compaction_suitable(zone, order, highest_zoneidx, 2313 zone_page_state(zone, NR_FREE_PAGES)); 2314 /* 2315 * fragmentation index determines if allocation failures are due to 2316 * low memory or external fragmentation 2317 * 2318 * index of -1000 would imply allocations might succeed depending on 2319 * watermarks, but we already failed the high-order watermark check 2320 * index towards 0 implies failure is due to lack of memory 2321 * index towards 1000 implies failure is due to fragmentation 2322 * 2323 * Only compact if a failure would be due to fragmentation. Also 2324 * ignore fragindex for non-costly orders where the alternative to 2325 * a successful reclaim/compaction is OOM. Fragindex and the 2326 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent 2327 * excessive compaction for costly orders, but it should not be at the 2328 * expense of system stability. 2329 */ 2330 if (suitable) { 2331 compact_result = COMPACT_CONTINUE; 2332 if (order > PAGE_ALLOC_COSTLY_ORDER) { 2333 int fragindex = fragmentation_index(zone, order); 2334 2335 if (fragindex >= 0 && 2336 fragindex <= sysctl_extfrag_threshold) { 2337 suitable = false; 2338 compact_result = COMPACT_NOT_SUITABLE_ZONE; 2339 } 2340 } 2341 } else { 2342 compact_result = COMPACT_SKIPPED; 2343 } 2344 2345 trace_mm_compaction_suitable(zone, order, compact_result); 2346 2347 return suitable; 2348 } 2349 2350 bool compaction_zonelist_suitable(struct alloc_context *ac, int order, 2351 int alloc_flags) 2352 { 2353 struct zone *zone; 2354 struct zoneref *z; 2355 2356 /* 2357 * Make sure at least one zone would pass __compaction_suitable if we continue 2358 * retrying the reclaim. 2359 */ 2360 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 2361 ac->highest_zoneidx, ac->nodemask) { 2362 unsigned long available; 2363 2364 /* 2365 * Do not consider all the reclaimable memory because we do not 2366 * want to trash just for a single high order allocation which 2367 * is even not guaranteed to appear even if __compaction_suitable 2368 * is happy about the watermark check. 2369 */ 2370 available = zone_reclaimable_pages(zone) / order; 2371 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 2372 if (__compaction_suitable(zone, order, ac->highest_zoneidx, 2373 available)) 2374 return true; 2375 } 2376 2377 return false; 2378 } 2379 2380 static enum compact_result 2381 compact_zone(struct compact_control *cc, struct capture_control *capc) 2382 { 2383 enum compact_result ret; 2384 unsigned long start_pfn = cc->zone->zone_start_pfn; 2385 unsigned long end_pfn = zone_end_pfn(cc->zone); 2386 unsigned long last_migrated_pfn; 2387 const bool sync = cc->mode != MIGRATE_ASYNC; 2388 bool update_cached; 2389 unsigned int nr_succeeded = 0; 2390 2391 /* 2392 * These counters track activities during zone compaction. Initialize 2393 * them before compacting a new zone. 2394 */ 2395 cc->total_migrate_scanned = 0; 2396 cc->total_free_scanned = 0; 2397 cc->nr_migratepages = 0; 2398 cc->nr_freepages = 0; 2399 INIT_LIST_HEAD(&cc->freepages); 2400 INIT_LIST_HEAD(&cc->migratepages); 2401 2402 cc->migratetype = gfp_migratetype(cc->gfp_mask); 2403 2404 if (!is_via_compact_memory(cc->order)) { 2405 unsigned long watermark; 2406 2407 /* Allocation can already succeed, nothing to do */ 2408 watermark = wmark_pages(cc->zone, 2409 cc->alloc_flags & ALLOC_WMARK_MASK); 2410 if (zone_watermark_ok(cc->zone, cc->order, watermark, 2411 cc->highest_zoneidx, cc->alloc_flags)) 2412 return COMPACT_SUCCESS; 2413 2414 /* Compaction is likely to fail */ 2415 if (!compaction_suitable(cc->zone, cc->order, 2416 cc->highest_zoneidx)) 2417 return COMPACT_SKIPPED; 2418 } 2419 2420 /* 2421 * Clear pageblock skip if there were failures recently and compaction 2422 * is about to be retried after being deferred. 2423 */ 2424 if (compaction_restarting(cc->zone, cc->order)) 2425 __reset_isolation_suitable(cc->zone); 2426 2427 /* 2428 * Setup to move all movable pages to the end of the zone. Used cached 2429 * information on where the scanners should start (unless we explicitly 2430 * want to compact the whole zone), but check that it is initialised 2431 * by ensuring the values are within zone boundaries. 2432 */ 2433 cc->fast_start_pfn = 0; 2434 if (cc->whole_zone) { 2435 cc->migrate_pfn = start_pfn; 2436 cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 2437 } else { 2438 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync]; 2439 cc->free_pfn = cc->zone->compact_cached_free_pfn; 2440 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) { 2441 cc->free_pfn = pageblock_start_pfn(end_pfn - 1); 2442 cc->zone->compact_cached_free_pfn = cc->free_pfn; 2443 } 2444 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) { 2445 cc->migrate_pfn = start_pfn; 2446 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; 2447 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; 2448 } 2449 2450 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn) 2451 cc->whole_zone = true; 2452 } 2453 2454 last_migrated_pfn = 0; 2455 2456 /* 2457 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on 2458 * the basis that some migrations will fail in ASYNC mode. However, 2459 * if the cached PFNs match and pageblocks are skipped due to having 2460 * no isolation candidates, then the sync state does not matter. 2461 * Until a pageblock with isolation candidates is found, keep the 2462 * cached PFNs in sync to avoid revisiting the same blocks. 2463 */ 2464 update_cached = !sync && 2465 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1]; 2466 2467 trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync); 2468 2469 /* lru_add_drain_all could be expensive with involving other CPUs */ 2470 lru_add_drain(); 2471 2472 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) { 2473 int err; 2474 unsigned long iteration_start_pfn = cc->migrate_pfn; 2475 2476 /* 2477 * Avoid multiple rescans of the same pageblock which can 2478 * happen if a page cannot be isolated (dirty/writeback in 2479 * async mode) or if the migrated pages are being allocated 2480 * before the pageblock is cleared. The first rescan will 2481 * capture the entire pageblock for migration. If it fails, 2482 * it'll be marked skip and scanning will proceed as normal. 2483 */ 2484 cc->finish_pageblock = false; 2485 if (pageblock_start_pfn(last_migrated_pfn) == 2486 pageblock_start_pfn(iteration_start_pfn)) { 2487 cc->finish_pageblock = true; 2488 } 2489 2490 rescan: 2491 switch (isolate_migratepages(cc)) { 2492 case ISOLATE_ABORT: 2493 ret = COMPACT_CONTENDED; 2494 putback_movable_pages(&cc->migratepages); 2495 cc->nr_migratepages = 0; 2496 goto out; 2497 case ISOLATE_NONE: 2498 if (update_cached) { 2499 cc->zone->compact_cached_migrate_pfn[1] = 2500 cc->zone->compact_cached_migrate_pfn[0]; 2501 } 2502 2503 /* 2504 * We haven't isolated and migrated anything, but 2505 * there might still be unflushed migrations from 2506 * previous cc->order aligned block. 2507 */ 2508 goto check_drain; 2509 case ISOLATE_SUCCESS: 2510 update_cached = false; 2511 last_migrated_pfn = max(cc->zone->zone_start_pfn, 2512 pageblock_start_pfn(cc->migrate_pfn - 1)); 2513 } 2514 2515 err = migrate_pages(&cc->migratepages, compaction_alloc, 2516 compaction_free, (unsigned long)cc, cc->mode, 2517 MR_COMPACTION, &nr_succeeded); 2518 2519 trace_mm_compaction_migratepages(cc, nr_succeeded); 2520 2521 /* All pages were either migrated or will be released */ 2522 cc->nr_migratepages = 0; 2523 if (err) { 2524 putback_movable_pages(&cc->migratepages); 2525 /* 2526 * migrate_pages() may return -ENOMEM when scanners meet 2527 * and we want compact_finished() to detect it 2528 */ 2529 if (err == -ENOMEM && !compact_scanners_met(cc)) { 2530 ret = COMPACT_CONTENDED; 2531 goto out; 2532 } 2533 /* 2534 * If an ASYNC or SYNC_LIGHT fails to migrate a page 2535 * within the pageblock_order-aligned block and 2536 * fast_find_migrateblock may be used then scan the 2537 * remainder of the pageblock. This will mark the 2538 * pageblock "skip" to avoid rescanning in the near 2539 * future. This will isolate more pages than necessary 2540 * for the request but avoid loops due to 2541 * fast_find_migrateblock revisiting blocks that were 2542 * recently partially scanned. 2543 */ 2544 if (!pageblock_aligned(cc->migrate_pfn) && 2545 !cc->ignore_skip_hint && !cc->finish_pageblock && 2546 (cc->mode < MIGRATE_SYNC)) { 2547 cc->finish_pageblock = true; 2548 2549 /* 2550 * Draining pcplists does not help THP if 2551 * any page failed to migrate. Even after 2552 * drain, the pageblock will not be free. 2553 */ 2554 if (cc->order == COMPACTION_HPAGE_ORDER) 2555 last_migrated_pfn = 0; 2556 2557 goto rescan; 2558 } 2559 } 2560 2561 /* Stop if a page has been captured */ 2562 if (capc && capc->page) { 2563 ret = COMPACT_SUCCESS; 2564 break; 2565 } 2566 2567 check_drain: 2568 /* 2569 * Has the migration scanner moved away from the previous 2570 * cc->order aligned block where we migrated from? If yes, 2571 * flush the pages that were freed, so that they can merge and 2572 * compact_finished() can detect immediately if allocation 2573 * would succeed. 2574 */ 2575 if (cc->order > 0 && last_migrated_pfn) { 2576 unsigned long current_block_start = 2577 block_start_pfn(cc->migrate_pfn, cc->order); 2578 2579 if (last_migrated_pfn < current_block_start) { 2580 lru_add_drain_cpu_zone(cc->zone); 2581 /* No more flushing until we migrate again */ 2582 last_migrated_pfn = 0; 2583 } 2584 } 2585 } 2586 2587 out: 2588 /* 2589 * Release free pages and update where the free scanner should restart, 2590 * so we don't leave any returned pages behind in the next attempt. 2591 */ 2592 if (cc->nr_freepages > 0) { 2593 unsigned long free_pfn = release_freepages(&cc->freepages); 2594 2595 cc->nr_freepages = 0; 2596 VM_BUG_ON(free_pfn == 0); 2597 /* The cached pfn is always the first in a pageblock */ 2598 free_pfn = pageblock_start_pfn(free_pfn); 2599 /* 2600 * Only go back, not forward. The cached pfn might have been 2601 * already reset to zone end in compact_finished() 2602 */ 2603 if (free_pfn > cc->zone->compact_cached_free_pfn) 2604 cc->zone->compact_cached_free_pfn = free_pfn; 2605 } 2606 2607 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned); 2608 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned); 2609 2610 trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret); 2611 2612 VM_BUG_ON(!list_empty(&cc->freepages)); 2613 VM_BUG_ON(!list_empty(&cc->migratepages)); 2614 2615 return ret; 2616 } 2617 2618 static enum compact_result compact_zone_order(struct zone *zone, int order, 2619 gfp_t gfp_mask, enum compact_priority prio, 2620 unsigned int alloc_flags, int highest_zoneidx, 2621 struct page **capture) 2622 { 2623 enum compact_result ret; 2624 struct compact_control cc = { 2625 .order = order, 2626 .search_order = order, 2627 .gfp_mask = gfp_mask, 2628 .zone = zone, 2629 .mode = (prio == COMPACT_PRIO_ASYNC) ? 2630 MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT, 2631 .alloc_flags = alloc_flags, 2632 .highest_zoneidx = highest_zoneidx, 2633 .direct_compaction = true, 2634 .whole_zone = (prio == MIN_COMPACT_PRIORITY), 2635 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), 2636 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) 2637 }; 2638 struct capture_control capc = { 2639 .cc = &cc, 2640 .page = NULL, 2641 }; 2642 2643 /* 2644 * Make sure the structs are really initialized before we expose the 2645 * capture control, in case we are interrupted and the interrupt handler 2646 * frees a page. 2647 */ 2648 barrier(); 2649 WRITE_ONCE(current->capture_control, &capc); 2650 2651 ret = compact_zone(&cc, &capc); 2652 2653 /* 2654 * Make sure we hide capture control first before we read the captured 2655 * page pointer, otherwise an interrupt could free and capture a page 2656 * and we would leak it. 2657 */ 2658 WRITE_ONCE(current->capture_control, NULL); 2659 *capture = READ_ONCE(capc.page); 2660 /* 2661 * Technically, it is also possible that compaction is skipped but 2662 * the page is still captured out of luck(IRQ came and freed the page). 2663 * Returning COMPACT_SUCCESS in such cases helps in properly accounting 2664 * the COMPACT[STALL|FAIL] when compaction is skipped. 2665 */ 2666 if (*capture) 2667 ret = COMPACT_SUCCESS; 2668 2669 return ret; 2670 } 2671 2672 /** 2673 * try_to_compact_pages - Direct compact to satisfy a high-order allocation 2674 * @gfp_mask: The GFP mask of the current allocation 2675 * @order: The order of the current allocation 2676 * @alloc_flags: The allocation flags of the current allocation 2677 * @ac: The context of current allocation 2678 * @prio: Determines how hard direct compaction should try to succeed 2679 * @capture: Pointer to free page created by compaction will be stored here 2680 * 2681 * This is the main entry point for direct page compaction. 2682 */ 2683 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, 2684 unsigned int alloc_flags, const struct alloc_context *ac, 2685 enum compact_priority prio, struct page **capture) 2686 { 2687 int may_perform_io = (__force int)(gfp_mask & __GFP_IO); 2688 struct zoneref *z; 2689 struct zone *zone; 2690 enum compact_result rc = COMPACT_SKIPPED; 2691 2692 /* 2693 * Check if the GFP flags allow compaction - GFP_NOIO is really 2694 * tricky context because the migration might require IO 2695 */ 2696 if (!may_perform_io) 2697 return COMPACT_SKIPPED; 2698 2699 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio); 2700 2701 /* Compact each zone in the list */ 2702 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 2703 ac->highest_zoneidx, ac->nodemask) { 2704 enum compact_result status; 2705 2706 if (prio > MIN_COMPACT_PRIORITY 2707 && compaction_deferred(zone, order)) { 2708 rc = max_t(enum compact_result, COMPACT_DEFERRED, rc); 2709 continue; 2710 } 2711 2712 status = compact_zone_order(zone, order, gfp_mask, prio, 2713 alloc_flags, ac->highest_zoneidx, capture); 2714 rc = max(status, rc); 2715 2716 /* The allocation should succeed, stop compacting */ 2717 if (status == COMPACT_SUCCESS) { 2718 /* 2719 * We think the allocation will succeed in this zone, 2720 * but it is not certain, hence the false. The caller 2721 * will repeat this with true if allocation indeed 2722 * succeeds in this zone. 2723 */ 2724 compaction_defer_reset(zone, order, false); 2725 2726 break; 2727 } 2728 2729 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE || 2730 status == COMPACT_PARTIAL_SKIPPED)) 2731 /* 2732 * We think that allocation won't succeed in this zone 2733 * so we defer compaction there. If it ends up 2734 * succeeding after all, it will be reset. 2735 */ 2736 defer_compaction(zone, order); 2737 2738 /* 2739 * We might have stopped compacting due to need_resched() in 2740 * async compaction, or due to a fatal signal detected. In that 2741 * case do not try further zones 2742 */ 2743 if ((prio == COMPACT_PRIO_ASYNC && need_resched()) 2744 || fatal_signal_pending(current)) 2745 break; 2746 } 2747 2748 return rc; 2749 } 2750 2751 /* 2752 * Compact all zones within a node till each zone's fragmentation score 2753 * reaches within proactive compaction thresholds (as determined by the 2754 * proactiveness tunable). 2755 * 2756 * It is possible that the function returns before reaching score targets 2757 * due to various back-off conditions, such as, contention on per-node or 2758 * per-zone locks. 2759 */ 2760 static void proactive_compact_node(pg_data_t *pgdat) 2761 { 2762 int zoneid; 2763 struct zone *zone; 2764 struct compact_control cc = { 2765 .order = -1, 2766 .mode = MIGRATE_SYNC_LIGHT, 2767 .ignore_skip_hint = true, 2768 .whole_zone = true, 2769 .gfp_mask = GFP_KERNEL, 2770 .proactive_compaction = true, 2771 }; 2772 2773 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 2774 zone = &pgdat->node_zones[zoneid]; 2775 if (!populated_zone(zone)) 2776 continue; 2777 2778 cc.zone = zone; 2779 2780 compact_zone(&cc, NULL); 2781 2782 count_compact_events(KCOMPACTD_MIGRATE_SCANNED, 2783 cc.total_migrate_scanned); 2784 count_compact_events(KCOMPACTD_FREE_SCANNED, 2785 cc.total_free_scanned); 2786 } 2787 } 2788 2789 /* Compact all zones within a node */ 2790 static void compact_node(int nid) 2791 { 2792 pg_data_t *pgdat = NODE_DATA(nid); 2793 int zoneid; 2794 struct zone *zone; 2795 struct compact_control cc = { 2796 .order = -1, 2797 .mode = MIGRATE_SYNC, 2798 .ignore_skip_hint = true, 2799 .whole_zone = true, 2800 .gfp_mask = GFP_KERNEL, 2801 }; 2802 2803 2804 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 2805 2806 zone = &pgdat->node_zones[zoneid]; 2807 if (!populated_zone(zone)) 2808 continue; 2809 2810 cc.zone = zone; 2811 2812 compact_zone(&cc, NULL); 2813 } 2814 } 2815 2816 /* Compact all nodes in the system */ 2817 static void compact_nodes(void) 2818 { 2819 int nid; 2820 2821 /* Flush pending updates to the LRU lists */ 2822 lru_add_drain_all(); 2823 2824 for_each_online_node(nid) 2825 compact_node(nid); 2826 } 2827 2828 static int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, 2829 void *buffer, size_t *length, loff_t *ppos) 2830 { 2831 int rc, nid; 2832 2833 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 2834 if (rc) 2835 return rc; 2836 2837 if (write && sysctl_compaction_proactiveness) { 2838 for_each_online_node(nid) { 2839 pg_data_t *pgdat = NODE_DATA(nid); 2840 2841 if (pgdat->proactive_compact_trigger) 2842 continue; 2843 2844 pgdat->proactive_compact_trigger = true; 2845 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1, 2846 pgdat->nr_zones - 1); 2847 wake_up_interruptible(&pgdat->kcompactd_wait); 2848 } 2849 } 2850 2851 return 0; 2852 } 2853 2854 /* 2855 * This is the entry point for compacting all nodes via 2856 * /proc/sys/vm/compact_memory 2857 */ 2858 static int sysctl_compaction_handler(struct ctl_table *table, int write, 2859 void *buffer, size_t *length, loff_t *ppos) 2860 { 2861 int ret; 2862 2863 ret = proc_dointvec(table, write, buffer, length, ppos); 2864 if (ret) 2865 return ret; 2866 2867 if (sysctl_compact_memory != 1) 2868 return -EINVAL; 2869 2870 if (write) 2871 compact_nodes(); 2872 2873 return 0; 2874 } 2875 2876 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 2877 static ssize_t compact_store(struct device *dev, 2878 struct device_attribute *attr, 2879 const char *buf, size_t count) 2880 { 2881 int nid = dev->id; 2882 2883 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { 2884 /* Flush pending updates to the LRU lists */ 2885 lru_add_drain_all(); 2886 2887 compact_node(nid); 2888 } 2889 2890 return count; 2891 } 2892 static DEVICE_ATTR_WO(compact); 2893 2894 int compaction_register_node(struct node *node) 2895 { 2896 return device_create_file(&node->dev, &dev_attr_compact); 2897 } 2898 2899 void compaction_unregister_node(struct node *node) 2900 { 2901 device_remove_file(&node->dev, &dev_attr_compact); 2902 } 2903 #endif /* CONFIG_SYSFS && CONFIG_NUMA */ 2904 2905 static inline bool kcompactd_work_requested(pg_data_t *pgdat) 2906 { 2907 return pgdat->kcompactd_max_order > 0 || kthread_should_stop() || 2908 pgdat->proactive_compact_trigger; 2909 } 2910 2911 static bool kcompactd_node_suitable(pg_data_t *pgdat) 2912 { 2913 int zoneid; 2914 struct zone *zone; 2915 enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx; 2916 2917 for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) { 2918 zone = &pgdat->node_zones[zoneid]; 2919 2920 if (!populated_zone(zone)) 2921 continue; 2922 2923 /* Allocation can already succeed, check other zones */ 2924 if (zone_watermark_ok(zone, pgdat->kcompactd_max_order, 2925 min_wmark_pages(zone), 2926 highest_zoneidx, 0)) 2927 continue; 2928 2929 if (compaction_suitable(zone, pgdat->kcompactd_max_order, 2930 highest_zoneidx)) 2931 return true; 2932 } 2933 2934 return false; 2935 } 2936 2937 static void kcompactd_do_work(pg_data_t *pgdat) 2938 { 2939 /* 2940 * With no special task, compact all zones so that a page of requested 2941 * order is allocatable. 2942 */ 2943 int zoneid; 2944 struct zone *zone; 2945 struct compact_control cc = { 2946 .order = pgdat->kcompactd_max_order, 2947 .search_order = pgdat->kcompactd_max_order, 2948 .highest_zoneidx = pgdat->kcompactd_highest_zoneidx, 2949 .mode = MIGRATE_SYNC_LIGHT, 2950 .ignore_skip_hint = false, 2951 .gfp_mask = GFP_KERNEL, 2952 }; 2953 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order, 2954 cc.highest_zoneidx); 2955 count_compact_event(KCOMPACTD_WAKE); 2956 2957 for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) { 2958 int status; 2959 2960 zone = &pgdat->node_zones[zoneid]; 2961 if (!populated_zone(zone)) 2962 continue; 2963 2964 if (compaction_deferred(zone, cc.order)) 2965 continue; 2966 2967 /* Allocation can already succeed, nothing to do */ 2968 if (zone_watermark_ok(zone, cc.order, 2969 min_wmark_pages(zone), zoneid, 0)) 2970 continue; 2971 2972 if (!compaction_suitable(zone, cc.order, zoneid)) 2973 continue; 2974 2975 if (kthread_should_stop()) 2976 return; 2977 2978 cc.zone = zone; 2979 status = compact_zone(&cc, NULL); 2980 2981 if (status == COMPACT_SUCCESS) { 2982 compaction_defer_reset(zone, cc.order, false); 2983 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) { 2984 /* 2985 * Buddy pages may become stranded on pcps that could 2986 * otherwise coalesce on the zone's free area for 2987 * order >= cc.order. This is ratelimited by the 2988 * upcoming deferral. 2989 */ 2990 drain_all_pages(zone); 2991 2992 /* 2993 * We use sync migration mode here, so we defer like 2994 * sync direct compaction does. 2995 */ 2996 defer_compaction(zone, cc.order); 2997 } 2998 2999 count_compact_events(KCOMPACTD_MIGRATE_SCANNED, 3000 cc.total_migrate_scanned); 3001 count_compact_events(KCOMPACTD_FREE_SCANNED, 3002 cc.total_free_scanned); 3003 } 3004 3005 /* 3006 * Regardless of success, we are done until woken up next. But remember 3007 * the requested order/highest_zoneidx in case it was higher/tighter 3008 * than our current ones 3009 */ 3010 if (pgdat->kcompactd_max_order <= cc.order) 3011 pgdat->kcompactd_max_order = 0; 3012 if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx) 3013 pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1; 3014 } 3015 3016 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx) 3017 { 3018 if (!order) 3019 return; 3020 3021 if (pgdat->kcompactd_max_order < order) 3022 pgdat->kcompactd_max_order = order; 3023 3024 if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx) 3025 pgdat->kcompactd_highest_zoneidx = highest_zoneidx; 3026 3027 /* 3028 * Pairs with implicit barrier in wait_event_freezable() 3029 * such that wakeups are not missed. 3030 */ 3031 if (!wq_has_sleeper(&pgdat->kcompactd_wait)) 3032 return; 3033 3034 if (!kcompactd_node_suitable(pgdat)) 3035 return; 3036 3037 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order, 3038 highest_zoneidx); 3039 wake_up_interruptible(&pgdat->kcompactd_wait); 3040 } 3041 3042 /* 3043 * The background compaction daemon, started as a kernel thread 3044 * from the init process. 3045 */ 3046 static int kcompactd(void *p) 3047 { 3048 pg_data_t *pgdat = (pg_data_t *)p; 3049 struct task_struct *tsk = current; 3050 long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC); 3051 long timeout = default_timeout; 3052 3053 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 3054 3055 if (!cpumask_empty(cpumask)) 3056 set_cpus_allowed_ptr(tsk, cpumask); 3057 3058 set_freezable(); 3059 3060 pgdat->kcompactd_max_order = 0; 3061 pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1; 3062 3063 while (!kthread_should_stop()) { 3064 unsigned long pflags; 3065 3066 /* 3067 * Avoid the unnecessary wakeup for proactive compaction 3068 * when it is disabled. 3069 */ 3070 if (!sysctl_compaction_proactiveness) 3071 timeout = MAX_SCHEDULE_TIMEOUT; 3072 trace_mm_compaction_kcompactd_sleep(pgdat->node_id); 3073 if (wait_event_freezable_timeout(pgdat->kcompactd_wait, 3074 kcompactd_work_requested(pgdat), timeout) && 3075 !pgdat->proactive_compact_trigger) { 3076 3077 psi_memstall_enter(&pflags); 3078 kcompactd_do_work(pgdat); 3079 psi_memstall_leave(&pflags); 3080 /* 3081 * Reset the timeout value. The defer timeout from 3082 * proactive compaction is lost here but that is fine 3083 * as the condition of the zone changing substantionally 3084 * then carrying on with the previous defer interval is 3085 * not useful. 3086 */ 3087 timeout = default_timeout; 3088 continue; 3089 } 3090 3091 /* 3092 * Start the proactive work with default timeout. Based 3093 * on the fragmentation score, this timeout is updated. 3094 */ 3095 timeout = default_timeout; 3096 if (should_proactive_compact_node(pgdat)) { 3097 unsigned int prev_score, score; 3098 3099 prev_score = fragmentation_score_node(pgdat); 3100 proactive_compact_node(pgdat); 3101 score = fragmentation_score_node(pgdat); 3102 /* 3103 * Defer proactive compaction if the fragmentation 3104 * score did not go down i.e. no progress made. 3105 */ 3106 if (unlikely(score >= prev_score)) 3107 timeout = 3108 default_timeout << COMPACT_MAX_DEFER_SHIFT; 3109 } 3110 if (unlikely(pgdat->proactive_compact_trigger)) 3111 pgdat->proactive_compact_trigger = false; 3112 } 3113 3114 return 0; 3115 } 3116 3117 /* 3118 * This kcompactd start function will be called by init and node-hot-add. 3119 * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added. 3120 */ 3121 void __meminit kcompactd_run(int nid) 3122 { 3123 pg_data_t *pgdat = NODE_DATA(nid); 3124 3125 if (pgdat->kcompactd) 3126 return; 3127 3128 pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid); 3129 if (IS_ERR(pgdat->kcompactd)) { 3130 pr_err("Failed to start kcompactd on node %d\n", nid); 3131 pgdat->kcompactd = NULL; 3132 } 3133 } 3134 3135 /* 3136 * Called by memory hotplug when all memory in a node is offlined. Caller must 3137 * be holding mem_hotplug_begin/done(). 3138 */ 3139 void __meminit kcompactd_stop(int nid) 3140 { 3141 struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd; 3142 3143 if (kcompactd) { 3144 kthread_stop(kcompactd); 3145 NODE_DATA(nid)->kcompactd = NULL; 3146 } 3147 } 3148 3149 /* 3150 * It's optimal to keep kcompactd on the same CPUs as their memory, but 3151 * not required for correctness. So if the last cpu in a node goes 3152 * away, we get changed to run anywhere: as the first one comes back, 3153 * restore their cpu bindings. 3154 */ 3155 static int kcompactd_cpu_online(unsigned int cpu) 3156 { 3157 int nid; 3158 3159 for_each_node_state(nid, N_MEMORY) { 3160 pg_data_t *pgdat = NODE_DATA(nid); 3161 const struct cpumask *mask; 3162 3163 mask = cpumask_of_node(pgdat->node_id); 3164 3165 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) 3166 /* One of our CPUs online: restore mask */ 3167 if (pgdat->kcompactd) 3168 set_cpus_allowed_ptr(pgdat->kcompactd, mask); 3169 } 3170 return 0; 3171 } 3172 3173 static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table, 3174 int write, void *buffer, size_t *lenp, loff_t *ppos) 3175 { 3176 int ret, old; 3177 3178 if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write) 3179 return proc_dointvec_minmax(table, write, buffer, lenp, ppos); 3180 3181 old = *(int *)table->data; 3182 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 3183 if (ret) 3184 return ret; 3185 if (old != *(int *)table->data) 3186 pr_warn_once("sysctl attribute %s changed by %s[%d]\n", 3187 table->procname, current->comm, 3188 task_pid_nr(current)); 3189 return ret; 3190 } 3191 3192 static struct ctl_table vm_compaction[] = { 3193 { 3194 .procname = "compact_memory", 3195 .data = &sysctl_compact_memory, 3196 .maxlen = sizeof(int), 3197 .mode = 0200, 3198 .proc_handler = sysctl_compaction_handler, 3199 }, 3200 { 3201 .procname = "compaction_proactiveness", 3202 .data = &sysctl_compaction_proactiveness, 3203 .maxlen = sizeof(sysctl_compaction_proactiveness), 3204 .mode = 0644, 3205 .proc_handler = compaction_proactiveness_sysctl_handler, 3206 .extra1 = SYSCTL_ZERO, 3207 .extra2 = SYSCTL_ONE_HUNDRED, 3208 }, 3209 { 3210 .procname = "extfrag_threshold", 3211 .data = &sysctl_extfrag_threshold, 3212 .maxlen = sizeof(int), 3213 .mode = 0644, 3214 .proc_handler = proc_dointvec_minmax, 3215 .extra1 = SYSCTL_ZERO, 3216 .extra2 = SYSCTL_ONE_THOUSAND, 3217 }, 3218 { 3219 .procname = "compact_unevictable_allowed", 3220 .data = &sysctl_compact_unevictable_allowed, 3221 .maxlen = sizeof(int), 3222 .mode = 0644, 3223 .proc_handler = proc_dointvec_minmax_warn_RT_change, 3224 .extra1 = SYSCTL_ZERO, 3225 .extra2 = SYSCTL_ONE, 3226 }, 3227 { } 3228 }; 3229 3230 static int __init kcompactd_init(void) 3231 { 3232 int nid; 3233 int ret; 3234 3235 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 3236 "mm/compaction:online", 3237 kcompactd_cpu_online, NULL); 3238 if (ret < 0) { 3239 pr_err("kcompactd: failed to register hotplug callbacks.\n"); 3240 return ret; 3241 } 3242 3243 for_each_node_state(nid, N_MEMORY) 3244 kcompactd_run(nid); 3245 register_sysctl_init("vm", vm_compaction); 3246 return 0; 3247 } 3248 subsys_initcall(kcompactd_init) 3249 3250 #endif /* CONFIG_COMPACTION */