개념 설명 전체 · v6.18.37 / mm/compaction.c

    1 // SPDX-License-Identifier: GPL-2.0
    2 /*
    3  * linux/mm/compaction.c
    4  *
    5  * Memory compaction for the reduction of external fragmentation. Note that
    6  * this heavily depends upon page migration to do all the real heavy
    7  * lifting
    8  *
    9  * Copyright IBM Corp. 2007-2010 Mel Gorman <[email protected]>
   10  */
   11 #include <linux/cpu.h>
   12 #include <linux/swap.h>
   13 #include <linux/migrate.h>
   14 #include <linux/compaction.h>
   15 #include <linux/mm_inline.h>
   16 #include <linux/sched/signal.h>
   17 #include <linux/backing-dev.h>
   18 #include <linux/sysctl.h>
   19 #include <linux/sysfs.h>
   20 #include <linux/page-isolation.h>
   21 #include <linux/kasan.h>
   22 #include <linux/kthread.h>
   23 #include <linux/freezer.h>
   24 #include <linux/page_owner.h>
   25 #include <linux/psi.h>
   26 #include <linux/cpuset.h>
   27 #include "internal.h"
   28 
   29 #ifdef CONFIG_COMPACTION
   30 /*
   31  * Fragmentation score check interval for proactive compaction purposes.
   32  */
   33 #define HPAGE_FRAG_CHECK_INTERVAL_MSEC	(500)
   34 
   35 static inline void count_compact_event(enum vm_event_item item)
   36 {
   37 	count_vm_event(item);
   38 }
   39 
   40 static inline void count_compact_events(enum vm_event_item item, long delta)
   41 {
   42 	count_vm_events(item, delta);
   43 }
   44 
   45 /*
   46  * order == -1 is expected when compacting proactively via
   47  * 1. /proc/sys/vm/compact_memory
   48  * 2. /sys/devices/system/node/nodex/compact
   49  * 3. /proc/sys/vm/compaction_proactiveness
   50  */
   51 static inline bool is_via_compact_memory(int order)
   52 {
   53 	return order == -1;
   54 }
   55 
   56 #else
   57 #define count_compact_event(item) do { } while (0)
   58 #define count_compact_events(item, delta) do { } while (0)
   59 static inline bool is_via_compact_memory(int order) { return false; }
   60 #endif
   61 
   62 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
   63 
   64 #define CREATE_TRACE_POINTS
   65 #include <trace/events/compaction.h>
   66 
   67 #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))
   68 #define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))
   69 
   70 /*
   71  * Page order with-respect-to which proactive compaction
   72  * calculates external fragmentation, which is used as
   73  * the "fragmentation score" of a node/zone.
   74  */
   75 #if defined CONFIG_TRANSPARENT_HUGEPAGE
   76 #define COMPACTION_HPAGE_ORDER	HPAGE_PMD_ORDER
   77 #elif defined CONFIG_HUGETLBFS
   78 #define COMPACTION_HPAGE_ORDER	HUGETLB_PAGE_ORDER
   79 #else
   80 #define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)
   81 #endif
   82 
   83 static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)
   84 {
   85 	post_alloc_hook(page, order, __GFP_MOVABLE);
   86 	set_page_refcounted(page);
   87 	return page;
   88 }
   89 #define mark_allocated(...)	alloc_hooks(mark_allocated_noprof(__VA_ARGS__))
   90 
   91 static unsigned long release_free_list(struct list_head *freepages)
   92 {
   93 	int order;
   94 	unsigned long high_pfn = 0;
   95 
   96 	for (order = 0; order < NR_PAGE_ORDERS; order++) {
   97 		struct page *page, *next;
   98 
   99 		list_for_each_entry_safe(page, next, &freepages[order], lru) {
  100 			unsigned long pfn = page_to_pfn(page);
  101 
  102 			list_del(&page->lru);
  103 			/*
  104 			 * Convert free pages into post allocation pages, so
  105 			 * that we can free them via __free_page.
  106 			 */
  107 			mark_allocated(page, order, __GFP_MOVABLE);
  108 			__free_pages(page, order);
  109 			if (pfn > high_pfn)
  110 				high_pfn = pfn;
  111 		}
  112 	}
  113 	return high_pfn;
  114 }
  115 
  116 #ifdef CONFIG_COMPACTION
  117 
  118 /* Do not skip compaction more than 64 times */
  119 #define COMPACT_MAX_DEFER_SHIFT 6
  120 
  121 /*
  122  * Compaction is deferred when compaction fails to result in a page
  123  * allocation success. 1 << compact_defer_shift, compactions are skipped up
  124  * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
  125  */
  126 static void defer_compaction(struct zone *zone, int order)
  127 {
  128 	zone->compact_considered = 0;
  129 	zone->compact_defer_shift++;
  130 
  131 	if (order < zone->compact_order_failed)
  132 		zone->compact_order_failed = order;
  133 
  134 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
  135 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
  136 
  137 	trace_mm_compaction_defer_compaction(zone, order);
  138 }
  139 
  140 /* Returns true if compaction should be skipped this time */
  141 static bool compaction_deferred(struct zone *zone, int order)
  142 {
  143 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;
  144 
  145 	if (order < zone->compact_order_failed)
  146 		return false;
  147 
  148 	/* Avoid possible overflow */
  149 	if (++zone->compact_considered >= defer_limit) {
  150 		zone->compact_considered = defer_limit;
  151 		return false;
  152 	}
  153 
  154 	trace_mm_compaction_deferred(zone, order);
  155 
  156 	return true;
  157 }
  158 
  159 /*
  160  * Update defer tracking counters after successful compaction of given order,
  161  * which means an allocation either succeeded (alloc_success == true) or is
  162  * expected to succeed.
  163  */
  164 void compaction_defer_reset(struct zone *zone, int order,
  165 		bool alloc_success)
  166 {
  167 	if (alloc_success) {
  168 		zone->compact_considered = 0;
  169 		zone->compact_defer_shift = 0;
  170 	}
  171 	if (order >= zone->compact_order_failed)
  172 		zone->compact_order_failed = order + 1;
  173 
  174 	trace_mm_compaction_defer_reset(zone, order);
  175 }
  176 
  177 /* Returns true if restarting compaction after many failures */
  178 static bool compaction_restarting(struct zone *zone, int order)
  179 {
  180 	if (order < zone->compact_order_failed)
  181 		return false;
  182 
  183 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
  184 		zone->compact_considered >= 1UL << zone->compact_defer_shift;
  185 }
  186 
  187 /* Returns true if the pageblock should be scanned for pages to isolate. */
  188 static inline bool isolation_suitable(struct compact_control *cc,
  189 					struct page *page)
  190 {
  191 	if (cc->ignore_skip_hint)
  192 		return true;
  193 
  194 	return !get_pageblock_skip(page);
  195 }
  196 
  197 static void reset_cached_positions(struct zone *zone)
  198 {
  199 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
  200 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
  201 	zone->compact_cached_free_pfn =
  202 				pageblock_start_pfn(zone_end_pfn(zone) - 1);
  203 }
  204 
  205 #ifdef CONFIG_SPARSEMEM
  206 /*
  207  * If the PFN falls into an offline section, return the start PFN of the
  208  * next online section. If the PFN falls into an online section or if
  209  * there is no next online section, return 0.
  210  */
  211 static unsigned long skip_offline_sections(unsigned long start_pfn)
  212 {
  213 	unsigned long start_nr = pfn_to_section_nr(start_pfn);
  214 
  215 	if (online_section_nr(start_nr))
  216 		return 0;
  217 
  218 	while (++start_nr <= __highest_present_section_nr) {
  219 		if (online_section_nr(start_nr))
  220 			return section_nr_to_pfn(start_nr);
  221 	}
  222 
  223 	return 0;
  224 }
  225 
  226 /*
  227  * If the PFN falls into an offline section, return the end PFN of the
  228  * next online section in reverse. If the PFN falls into an online section
  229  * or if there is no next online section in reverse, return 0.
  230  */
  231 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
  232 {
  233 	unsigned long start_nr = pfn_to_section_nr(start_pfn);
  234 
  235 	if (!start_nr || online_section_nr(start_nr))
  236 		return 0;
  237 
  238 	while (start_nr-- > 0) {
  239 		if (online_section_nr(start_nr))
  240 			return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;
  241 	}
  242 
  243 	return 0;
  244 }
  245 #else
  246 static unsigned long skip_offline_sections(unsigned long start_pfn)
  247 {
  248 	return 0;
  249 }
  250 
  251 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
  252 {
  253 	return 0;
  254 }
  255 #endif
  256 
  257 /*
  258  * Compound pages of >= pageblock_order should consistently be skipped until
  259  * released. It is always pointless to compact pages of such order (if they are
  260  * migratable), and the pageblocks they occupy cannot contain any free pages.
  261  */
  262 static bool pageblock_skip_persistent(struct page *page)
  263 {
  264 	if (!PageCompound(page))
  265 		return false;
  266 
  267 	page = compound_head(page);
  268 
  269 	if (compound_order(page) >= pageblock_order)
  270 		return true;
  271 
  272 	return false;
  273 }
  274 
  275 static bool
  276 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
  277 							bool check_target)
  278 {
  279 	struct page *page = pfn_to_online_page(pfn);
  280 	struct page *block_page;
  281 	struct page *end_page;
  282 	unsigned long block_pfn;
  283 
  284 	if (!page)
  285 		return false;
  286 	if (zone != page_zone(page))
  287 		return false;
  288 	if (pageblock_skip_persistent(page))
  289 		return false;
  290 
  291 	/*
  292 	 * If skip is already cleared do no further checking once the
  293 	 * restart points have been set.
  294 	 */
  295 	if (check_source && check_target && !get_pageblock_skip(page))
  296 		return true;
  297 
  298 	/*
  299 	 * If clearing skip for the target scanner, do not select a
  300 	 * non-movable pageblock as the starting point.
  301 	 */
  302 	if (!check_source && check_target &&
  303 	    get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
  304 		return false;
  305 
  306 	/* Ensure the start of the pageblock or zone is online and valid */
  307 	block_pfn = pageblock_start_pfn(pfn);
  308 	block_pfn = max(block_pfn, zone->zone_start_pfn);
  309 	block_page = pfn_to_online_page(block_pfn);
  310 	if (block_page) {
  311 		page = block_page;
  312 		pfn = block_pfn;
  313 	}
  314 
  315 	/* Ensure the end of the pageblock or zone is online and valid */
  316 	block_pfn = pageblock_end_pfn(pfn) - 1;
  317 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
  318 	end_page = pfn_to_online_page(block_pfn);
  319 	if (!end_page)
  320 		return false;
  321 
  322 	/*
  323 	 * Only clear the hint if a sample indicates there is either a
  324 	 * free page or an LRU page in the block. One or other condition
  325 	 * is necessary for the block to be a migration source/target.
  326 	 */
  327 	do {
  328 		if (check_source && PageLRU(page)) {
  329 			clear_pageblock_skip(page);
  330 			return true;
  331 		}
  332 
  333 		if (check_target && PageBuddy(page)) {
  334 			clear_pageblock_skip(page);
  335 			return true;
  336 		}
  337 
  338 		page += (1 << PAGE_ALLOC_COSTLY_ORDER);
  339 	} while (page <= end_page);
  340 
  341 	return false;
  342 }
  343 
  344 /*
  345  * This function is called to clear all cached information on pageblocks that
  346  * should be skipped for page isolation when the migrate and free page scanner
  347  * meet.
  348  */
  349 static void __reset_isolation_suitable(struct zone *zone)
  350 {
  351 	unsigned long migrate_pfn = zone->zone_start_pfn;
  352 	unsigned long free_pfn = zone_end_pfn(zone) - 1;
  353 	unsigned long reset_migrate = free_pfn;
  354 	unsigned long reset_free = migrate_pfn;
  355 	bool source_set = false;
  356 	bool free_set = false;
  357 
  358 	/* Only flush if a full compaction finished recently */
  359 	if (!zone->compact_blockskip_flush)
  360 		return;
  361 
  362 	zone->compact_blockskip_flush = false;
  363 
  364 	/*
  365 	 * Walk the zone and update pageblock skip information. Source looks
  366 	 * for PageLRU while target looks for PageBuddy. When the scanner
  367 	 * is found, both PageBuddy and PageLRU are checked as the pageblock
  368 	 * is suitable as both source and target.
  369 	 */
  370 	for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
  371 					free_pfn -= pageblock_nr_pages) {
  372 		cond_resched();
  373 
  374 		/* Update the migrate PFN */
  375 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
  376 		    migrate_pfn < reset_migrate) {
  377 			source_set = true;
  378 			reset_migrate = migrate_pfn;
  379 			zone->compact_init_migrate_pfn = reset_migrate;
  380 			zone->compact_cached_migrate_pfn[0] = reset_migrate;
  381 			zone->compact_cached_migrate_pfn[1] = reset_migrate;
  382 		}
  383 
  384 		/* Update the free PFN */
  385 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
  386 		    free_pfn > reset_free) {
  387 			free_set = true;
  388 			reset_free = free_pfn;
  389 			zone->compact_init_free_pfn = reset_free;
  390 			zone->compact_cached_free_pfn = reset_free;
  391 		}
  392 	}
  393 
  394 	/* Leave no distance if no suitable block was reset */
  395 	if (reset_migrate >= reset_free) {
  396 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;
  397 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;
  398 		zone->compact_cached_free_pfn = free_pfn;
  399 	}
  400 }
  401 
  402 void reset_isolation_suitable(pg_data_t *pgdat)
  403 {
  404 	int zoneid;
  405 
  406 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
  407 		struct zone *zone = &pgdat->node_zones[zoneid];
  408 		if (!populated_zone(zone))
  409 			continue;
  410 
  411 		__reset_isolation_suitable(zone);
  412 	}
  413 }
  414 
  415 /*
  416  * Sets the pageblock skip bit if it was clear. Note that this is a hint as
  417  * locks are not required for read/writers. Returns true if it was already set.
  418  */
  419 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
  420 {
  421 	bool skip;
  422 
  423 	/* Do not update if skip hint is being ignored */
  424 	if (cc->ignore_skip_hint)
  425 		return false;
  426 
  427 	skip = get_pageblock_skip(page);
  428 	if (!skip && !cc->no_set_skip_hint)
  429 		set_pageblock_skip(page);
  430 
  431 	return skip;
  432 }
  433 
  434 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
  435 {
  436 	struct zone *zone = cc->zone;
  437 
  438 	/* Set for isolation rather than compaction */
  439 	if (cc->no_set_skip_hint)
  440 		return;
  441 
  442 	pfn = pageblock_end_pfn(pfn);
  443 
  444 	/* Update where async and sync compaction should restart */
  445 	if (pfn > zone->compact_cached_migrate_pfn[0])
  446 		zone->compact_cached_migrate_pfn[0] = pfn;
  447 	if (cc->mode != MIGRATE_ASYNC &&
  448 	    pfn > zone->compact_cached_migrate_pfn[1])
  449 		zone->compact_cached_migrate_pfn[1] = pfn;
  450 }
  451 
  452 /*
  453  * If no pages were isolated then mark this pageblock to be skipped in the
  454  * future. The information is later cleared by __reset_isolation_suitable().
  455  */
  456 static void update_pageblock_skip(struct compact_control *cc,
  457 			struct page *page, unsigned long pfn)
  458 {
  459 	struct zone *zone = cc->zone;
  460 
  461 	if (cc->no_set_skip_hint)
  462 		return;
  463 
  464 	set_pageblock_skip(page);
  465 
  466 	if (pfn < zone->compact_cached_free_pfn)
  467 		zone->compact_cached_free_pfn = pfn;
  468 }
  469 #else
  470 static inline bool isolation_suitable(struct compact_control *cc,
  471 					struct page *page)
  472 {
  473 	return true;
  474 }
  475 
  476 static inline bool pageblock_skip_persistent(struct page *page)
  477 {
  478 	return false;
  479 }
  480 
  481 static inline void update_pageblock_skip(struct compact_control *cc,
  482 			struct page *page, unsigned long pfn)
  483 {
  484 }
  485 
  486 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
  487 {
  488 }
  489 
  490 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
  491 {
  492 	return false;
  493 }
  494 #endif /* CONFIG_COMPACTION */
  495 
  496 /*
  497  * Compaction requires the taking of some coarse locks that are potentially
  498  * very heavily contended. For async compaction, trylock and record if the
  499  * lock is contended. The lock will still be acquired but compaction will
  500  * abort when the current block is finished regardless of success rate.
  501  * Sync compaction acquires the lock.
  502  *
  503  * Always returns true which makes it easier to track lock state in callers.
  504  */
  505 static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
  506 						struct compact_control *cc)
  507 	__acquires(lock)
  508 {
  509 	/* Track if the lock is contended in async mode */
  510 	if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
  511 		if (spin_trylock_irqsave(lock, *flags))
  512 			return true;
  513 
  514 		cc->contended = true;
  515 	}
  516 
  517 	spin_lock_irqsave(lock, *flags);
  518 	return true;
  519 }
  520 
  521 /*
  522  * Compaction requires the taking of some coarse locks that are potentially
  523  * very heavily contended. The lock should be periodically unlocked to avoid
  524  * having disabled IRQs for a long time, even when there is nobody waiting on
  525  * the lock. It might also be that allowing the IRQs will result in
  526  * need_resched() becoming true. If scheduling is needed, compaction schedules.
  527  * Either compaction type will also abort if a fatal signal is pending.
  528  * In either case if the lock was locked, it is dropped and not regained.
  529  *
  530  * Returns true if compaction should abort due to fatal signal pending.
  531  * Returns false when compaction can continue.
  532  */
  533 static bool compact_unlock_should_abort(spinlock_t *lock,
  534 		unsigned long flags, bool *locked, struct compact_control *cc)
  535 {
  536 	if (*locked) {
  537 		spin_unlock_irqrestore(lock, flags);
  538 		*locked = false;
  539 	}
  540 
  541 	if (fatal_signal_pending(current)) {
  542 		cc->contended = true;
  543 		return true;
  544 	}
  545 
  546 	cond_resched();
  547 
  548 	return false;
  549 }
  550 
  551 /*
  552  * Isolate free pages onto a private freelist. If @strict is true, will abort
  553  * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
  554  * (even though it may still end up isolating some pages).
  555  */
  556 static unsigned long isolate_freepages_block(struct compact_control *cc,
  557 				unsigned long *start_pfn,
  558 				unsigned long end_pfn,
  559 				struct list_head *freelist,
  560 				unsigned int stride,
  561 				bool strict)
  562 {
  563 	int nr_scanned = 0, total_isolated = 0;
  564 	struct page *page;
  565 	unsigned long flags = 0;
  566 	bool locked = false;
  567 	unsigned long blockpfn = *start_pfn;
  568 	unsigned int order;
  569 
  570 	/* Strict mode is for isolation, speed is secondary */
  571 	if (strict)
  572 		stride = 1;
  573 
  574 	page = pfn_to_page(blockpfn);
  575 
  576 	/* Isolate free pages. */
  577 	for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {
  578 		int isolated;
  579 
  580 		/*
  581 		 * Periodically drop the lock (if held) regardless of its
  582 		 * contention, to give chance to IRQs. Abort if fatal signal
  583 		 * pending.
  584 		 */
  585 		if (!(blockpfn % COMPACT_CLUSTER_MAX)
  586 		    && compact_unlock_should_abort(&cc->zone->lock, flags,
  587 								&locked, cc))
  588 			break;
  589 
  590 		nr_scanned++;
  591 
  592 		/*
  593 		 * For compound pages such as THP and hugetlbfs, we can save
  594 		 * potentially a lot of iterations if we skip them at once.
  595 		 * The check is racy, but we can consider only valid values
  596 		 * and the only danger is skipping too much.
  597 		 */
  598 		if (PageCompound(page)) {
  599 			const unsigned int order = compound_order(page);
  600 
  601 			if ((order <= MAX_PAGE_ORDER) &&
  602 			    (blockpfn + (1UL << order) <= end_pfn)) {
  603 				blockpfn += (1UL << order) - 1;
  604 				page += (1UL << order) - 1;
  605 				nr_scanned += (1UL << order) - 1;
  606 			}
  607 
  608 			goto isolate_fail;
  609 		}
  610 
  611 		if (!PageBuddy(page))
  612 			goto isolate_fail;
  613 
  614 		/* If we already hold the lock, we can skip some rechecking. */
  615 		if (!locked) {
  616 			locked = compact_lock_irqsave(&cc->zone->lock,
  617 								&flags, cc);
  618 
  619 			/* Recheck this is a buddy page under lock */
  620 			if (!PageBuddy(page))
  621 				goto isolate_fail;
  622 		}
  623 
  624 		/* Found a free page, will break it into order-0 pages */
  625 		order = buddy_order(page);
  626 		isolated = __isolate_free_page(page, order);
  627 		if (!isolated)
  628 			break;
  629 		set_page_private(page, order);
  630 
  631 		nr_scanned += isolated - 1;
  632 		total_isolated += isolated;
  633 		cc->nr_freepages += isolated;
  634 		list_add_tail(&page->lru, &freelist[order]);
  635 
  636 		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
  637 			blockpfn += isolated;
  638 			break;
  639 		}
  640 		/* Advance to the end of split page */
  641 		blockpfn += isolated - 1;
  642 		page += isolated - 1;
  643 		continue;
  644 
  645 isolate_fail:
  646 		if (strict)
  647 			break;
  648 
  649 	}
  650 
  651 	if (locked)
  652 		spin_unlock_irqrestore(&cc->zone->lock, flags);
  653 
  654 	/*
  655 	 * Be careful to not go outside of the pageblock.
  656 	 */
  657 	if (unlikely(blockpfn > end_pfn))
  658 		blockpfn = end_pfn;
  659 
  660 	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
  661 					nr_scanned, total_isolated);
  662 
  663 	/* Record how far we have got within the block */
  664 	*start_pfn = blockpfn;
  665 
  666 	/*
  667 	 * If strict isolation is requested by CMA then check that all the
  668 	 * pages requested were isolated. If there were any failures, 0 is
  669 	 * returned and CMA will fail.
  670 	 */
  671 	if (strict && blockpfn < end_pfn)
  672 		total_isolated = 0;
  673 
  674 	cc->total_free_scanned += nr_scanned;
  675 	if (total_isolated)
  676 		count_compact_events(COMPACTISOLATED, total_isolated);
  677 	return total_isolated;
  678 }
  679 
  680 /**
  681  * isolate_freepages_range() - isolate free pages.
  682  * @cc:        Compaction control structure.
  683  * @start_pfn: The first PFN to start isolating.
  684  * @end_pfn:   The one-past-last PFN.
  685  *
  686  * Non-free pages, invalid PFNs, or zone boundaries within the
  687  * [start_pfn, end_pfn) range are considered errors, cause function to
  688  * undo its actions and return zero. cc->freepages[] are empty.
  689  *
  690  * Otherwise, function returns one-past-the-last PFN of isolated page
  691  * (which may be greater then end_pfn if end fell in a middle of
  692  * a free page). cc->freepages[] contain free pages isolated.
  693  */
  694 unsigned long
  695 isolate_freepages_range(struct compact_control *cc,
  696 			unsigned long start_pfn, unsigned long end_pfn)
  697 {
  698 	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
  699 	int order;
  700 
  701 	for (order = 0; order < NR_PAGE_ORDERS; order++)
  702 		INIT_LIST_HEAD(&cc->freepages[order]);
  703 
  704 	pfn = start_pfn;
  705 	block_start_pfn = pageblock_start_pfn(pfn);
  706 	if (block_start_pfn < cc->zone->zone_start_pfn)
  707 		block_start_pfn = cc->zone->zone_start_pfn;
  708 	block_end_pfn = pageblock_end_pfn(pfn);
  709 
  710 	for (; pfn < end_pfn; pfn += isolated,
  711 				block_start_pfn = block_end_pfn,
  712 				block_end_pfn += pageblock_nr_pages) {
  713 		/* Protect pfn from changing by isolate_freepages_block */
  714 		unsigned long isolate_start_pfn = pfn;
  715 
  716 		/*
  717 		 * pfn could pass the block_end_pfn if isolated freepage
  718 		 * is more than pageblock order. In this case, we adjust
  719 		 * scanning range to right one.
  720 		 */
  721 		if (pfn >= block_end_pfn) {
  722 			block_start_pfn = pageblock_start_pfn(pfn);
  723 			block_end_pfn = pageblock_end_pfn(pfn);
  724 		}
  725 
  726 		block_end_pfn = min(block_end_pfn, end_pfn);
  727 
  728 		if (!pageblock_pfn_to_page(block_start_pfn,
  729 					block_end_pfn, cc->zone))
  730 			break;
  731 
  732 		isolated = isolate_freepages_block(cc, &isolate_start_pfn,
  733 					block_end_pfn, cc->freepages, 0, true);
  734 
  735 		/*
  736 		 * In strict mode, isolate_freepages_block() returns 0 if
  737 		 * there are any holes in the block (ie. invalid PFNs or
  738 		 * non-free pages).
  739 		 */
  740 		if (!isolated)
  741 			break;
  742 
  743 		/*
  744 		 * If we managed to isolate pages, it is always (1 << n) *
  745 		 * pageblock_nr_pages for some non-negative n.  (Max order
  746 		 * page may span two pageblocks).
  747 		 */
  748 	}
  749 
  750 	if (pfn < end_pfn) {
  751 		/* Loop terminated early, cleanup. */
  752 		release_free_list(cc->freepages);
  753 		return 0;
  754 	}
  755 
  756 	/* We don't use freelists for anything. */
  757 	return pfn;
  758 }
  759 
  760 /* Similar to reclaim, but different enough that they don't share logic */
  761 static bool too_many_isolated(struct compact_control *cc)
  762 {
  763 	pg_data_t *pgdat = cc->zone->zone_pgdat;
  764 	bool too_many;
  765 
  766 	unsigned long active, inactive, isolated;
  767 
  768 	inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
  769 			node_page_state(pgdat, NR_INACTIVE_ANON);
  770 	active = node_page_state(pgdat, NR_ACTIVE_FILE) +
  771 			node_page_state(pgdat, NR_ACTIVE_ANON);
  772 	isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
  773 			node_page_state(pgdat, NR_ISOLATED_ANON);
  774 
  775 	/*
  776 	 * Allow GFP_NOFS to isolate past the limit set for regular
  777 	 * compaction runs. This prevents an ABBA deadlock when other
  778 	 * compactors have already isolated to the limit, but are
  779 	 * blocked on filesystem locks held by the GFP_NOFS thread.
  780 	 */
  781 	if (cc->gfp_mask & __GFP_FS) {
  782 		inactive >>= 3;
  783 		active >>= 3;
  784 	}
  785 
  786 	too_many = isolated > (inactive + active) / 2;
  787 	if (!too_many)
  788 		wake_throttle_isolated(pgdat);
  789 
  790 	return too_many;
  791 }
  792 
  793 /**
  794  * skip_isolation_on_order() - determine when to skip folio isolation based on
  795  *			       folio order and compaction target order
  796  * @order:		to-be-isolated folio order
  797  * @target_order:	compaction target order
  798  *
  799  * This avoids unnecessary folio isolations during compaction.
  800  */
  801 static bool skip_isolation_on_order(int order, int target_order)
  802 {
  803 	/*
  804 	 * Unless we are performing global compaction (i.e.,
  805 	 * is_via_compact_memory), skip any folios that are larger than the
  806 	 * target order: we wouldn't be here if we'd have a free folio with
  807 	 * the desired target_order, so migrating this folio would likely fail
  808 	 * later.
  809 	 */
  810 	if (!is_via_compact_memory(target_order) && order >= target_order)
  811 		return true;
  812 	/*
  813 	 * We limit memory compaction to pageblocks and won't try
  814 	 * creating free blocks of memory that are larger than that.
  815 	 */
  816 	return order >= pageblock_order;
  817 }
  818 
  819 /**
  820  * isolate_migratepages_block() - isolate all migrate-able pages within
  821  *				  a single pageblock
  822  * @cc:		Compaction control structure.
  823  * @low_pfn:	The first PFN to isolate
  824  * @end_pfn:	The one-past-the-last PFN to isolate, within same pageblock
  825  * @mode:	Isolation mode to be used.
  826  *
  827  * Isolate all pages that can be migrated from the range specified by
  828  * [low_pfn, end_pfn). The range is expected to be within same pageblock.
  829  * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
  830  * -ENOMEM in case we could not allocate a page, or 0.
  831  * cc->migrate_pfn will contain the next pfn to scan.
  832  *
  833  * The pages are isolated on cc->migratepages list (not required to be empty),
  834  * and cc->nr_migratepages is updated accordingly.
  835  */
  836 static int
  837 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
  838 			unsigned long end_pfn, isolate_mode_t mode)
  839 {
  840 	pg_data_t *pgdat = cc->zone->zone_pgdat;
  841 	unsigned long nr_scanned = 0, nr_isolated = 0;
  842 	struct lruvec *lruvec;
  843 	unsigned long flags = 0;
  844 	struct lruvec *locked = NULL;
  845 	struct folio *folio = NULL;
  846 	struct page *page = NULL, *valid_page = NULL;
  847 	struct address_space *mapping;
  848 	unsigned long start_pfn = low_pfn;
  849 	bool skip_on_failure = false;
  850 	unsigned long next_skip_pfn = 0;
  851 	bool skip_updated = false;
  852 	int ret = 0;
  853 
  854 	cc->migrate_pfn = low_pfn;
  855 
  856 	/*
  857 	 * Ensure that there are not too many pages isolated from the LRU
  858 	 * list by either parallel reclaimers or compaction. If there are,
  859 	 * delay for some time until fewer pages are isolated
  860 	 */
  861 	while (unlikely(too_many_isolated(cc))) {
  862 		/* stop isolation if there are still pages not migrated */
  863 		if (cc->nr_migratepages)
  864 			return -EAGAIN;
  865 
  866 		/* async migration should just abort */
  867 		if (cc->mode == MIGRATE_ASYNC)
  868 			return -EAGAIN;
  869 
  870 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
  871 
  872 		if (fatal_signal_pending(current))
  873 			return -EINTR;
  874 	}
  875 
  876 	cond_resched();
  877 
  878 	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
  879 		skip_on_failure = true;
  880 		next_skip_pfn = block_end_pfn(low_pfn, cc->order);
  881 	}
  882 
  883 	/* Time to isolate some pages for migration */
  884 	for (; low_pfn < end_pfn; low_pfn++) {
  885 		bool is_dirty, is_unevictable;
  886 
  887 		if (skip_on_failure && low_pfn >= next_skip_pfn) {
  888 			/*
  889 			 * We have isolated all migration candidates in the
  890 			 * previous order-aligned block, and did not skip it due
  891 			 * to failure. We should migrate the pages now and
  892 			 * hopefully succeed compaction.
  893 			 */
  894 			if (nr_isolated)
  895 				break;
  896 
  897 			/*
  898 			 * We failed to isolate in the previous order-aligned
  899 			 * block. Set the new boundary to the end of the
  900 			 * current block. Note we can't simply increase
  901 			 * next_skip_pfn by 1 << order, as low_pfn might have
  902 			 * been incremented by a higher number due to skipping
  903 			 * a compound or a high-order buddy page in the
  904 			 * previous loop iteration.
  905 			 */
  906 			next_skip_pfn = block_end_pfn(low_pfn, cc->order);
  907 		}
  908 
  909 		/*
  910 		 * Periodically drop the lock (if held) regardless of its
  911 		 * contention, to give chance to IRQs. Abort completely if
  912 		 * a fatal signal is pending.
  913 		 */
  914 		if (!(low_pfn % COMPACT_CLUSTER_MAX)) {
  915 			if (locked) {
  916 				unlock_page_lruvec_irqrestore(locked, flags);
  917 				locked = NULL;
  918 			}
  919 
  920 			if (fatal_signal_pending(current)) {
  921 				cc->contended = true;
  922 				ret = -EINTR;
  923 
  924 				goto fatal_pending;
  925 			}
  926 
  927 			cond_resched();
  928 		}
  929 
  930 		nr_scanned++;
  931 
  932 		page = pfn_to_page(low_pfn);
  933 
  934 		/*
  935 		 * Check if the pageblock has already been marked skipped.
  936 		 * Only the first PFN is checked as the caller isolates
  937 		 * COMPACT_CLUSTER_MAX at a time so the second call must
  938 		 * not falsely conclude that the block should be skipped.
  939 		 */
  940 		if (!valid_page && (pageblock_aligned(low_pfn) ||
  941 				    low_pfn == cc->zone->zone_start_pfn)) {
  942 			if (!isolation_suitable(cc, page)) {
  943 				low_pfn = end_pfn;
  944 				folio = NULL;
  945 				goto isolate_abort;
  946 			}
  947 			valid_page = page;
  948 		}
  949 
  950 		if (PageHuge(page)) {
  951 			const unsigned int order = compound_order(page);
  952 			/*
  953 			 * skip hugetlbfs if we are not compacting for pages
  954 			 * bigger than its order. THPs and other compound pages
  955 			 * are handled below.
  956 			 */
  957 			if (!cc->alloc_contig) {
  958 
  959 				if (order <= MAX_PAGE_ORDER) {
  960 					low_pfn += (1UL << order) - 1;
  961 					nr_scanned += (1UL << order) - 1;
  962 				}
  963 				goto isolate_fail;
  964 			}
  965 			/* for alloc_contig case */
  966 			if (locked) {
  967 				unlock_page_lruvec_irqrestore(locked, flags);
  968 				locked = NULL;
  969 			}
  970 
  971 			folio = page_folio(page);
  972 			ret = isolate_or_dissolve_huge_folio(folio, &cc->migratepages);
  973 
  974 			/*
  975 			 * Fail isolation in case isolate_or_dissolve_huge_folio()
  976 			 * reports an error. In case of -ENOMEM, abort right away.
  977 			 */
  978 			if (ret < 0) {
  979 				 /* Do not report -EBUSY down the chain */
  980 				if (ret == -EBUSY)
  981 					ret = 0;
  982 				low_pfn += (1UL << order) - 1;
  983 				nr_scanned += (1UL << order) - 1;
  984 				goto isolate_fail;
  985 			}
  986 
  987 			if (folio_test_hugetlb(folio)) {
  988 				/*
  989 				 * Hugepage was successfully isolated and placed
  990 				 * on the cc->migratepages list.
  991 				 */
  992 				low_pfn += folio_nr_pages(folio) - folio_page_idx(folio, page) - 1;
  993 				goto isolate_success_no_list;
  994 			}
  995 
  996 			/*
  997 			 * Ok, the hugepage was dissolved. Now these pages are
  998 			 * Buddy and cannot be re-allocated because they are
  999 			 * isolated. Fall-through as the check below handles
 1000 			 * Buddy pages.
 1001 			 */
 1002 		}
 1003 
 1004 		/*
 1005 		 * Skip if free. We read page order here without zone lock
 1006 		 * which is generally unsafe, but the race window is small and
 1007 		 * the worst thing that can happen is that we skip some
 1008 		 * potential isolation targets.
 1009 		 */
 1010 		if (PageBuddy(page)) {
 1011 			unsigned long freepage_order = buddy_order_unsafe(page);
 1012 
 1013 			/*
 1014 			 * Without lock, we cannot be sure that what we got is
 1015 			 * a valid page order. Consider only values in the
 1016 			 * valid order range to prevent low_pfn overflow.
 1017 			 */
 1018 			if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {
 1019 				low_pfn += (1UL << freepage_order) - 1;
 1020 				nr_scanned += (1UL << freepage_order) - 1;
 1021 			}
 1022 			continue;
 1023 		}
 1024 
 1025 		/*
 1026 		 * Regardless of being on LRU, compound pages such as THP
 1027 		 * (hugetlbfs is handled above) are not to be compacted unless
 1028 		 * we are attempting an allocation larger than the compound
 1029 		 * page size. We can potentially save a lot of iterations if we
 1030 		 * skip them at once. The check is racy, but we can consider
 1031 		 * only valid values and the only danger is skipping too much.
 1032 		 */
 1033 		if (PageCompound(page) && !cc->alloc_contig) {
 1034 			const unsigned int order = compound_order(page);
 1035 
 1036 			/* Skip based on page order and compaction target order. */
 1037 			if (skip_isolation_on_order(order, cc->order)) {
 1038 				if (order <= MAX_PAGE_ORDER) {
 1039 					low_pfn += (1UL << order) - 1;
 1040 					nr_scanned += (1UL << order) - 1;
 1041 				}
 1042 				goto isolate_fail;
 1043 			}
 1044 		}
 1045 
 1046 		/*
 1047 		 * Check may be lockless but that's ok as we recheck later.
 1048 		 * It's possible to migrate LRU and non-lru movable pages.
 1049 		 * Skip any other type of page
 1050 		 */
 1051 		if (!PageLRU(page)) {
 1052 			/* Isolation code will deal with any races. */
 1053 			if (unlikely(page_has_movable_ops(page)) &&
 1054 			    !PageMovableOpsIsolated(page)) {
 1055 				if (locked) {
 1056 					unlock_page_lruvec_irqrestore(locked, flags);
 1057 					locked = NULL;
 1058 				}
 1059 
 1060 				if (isolate_movable_ops_page(page, mode)) {
 1061 					folio = page_folio(page);
 1062 					goto isolate_success;
 1063 				}
 1064 			}
 1065 
 1066 			goto isolate_fail;
 1067 		}
 1068 
 1069 		/*
 1070 		 * Be careful not to clear PageLRU until after we're
 1071 		 * sure the page is not being freed elsewhere -- the
 1072 		 * page release code relies on it.
 1073 		 */
 1074 		folio = folio_get_nontail_page(page);
 1075 		if (unlikely(!folio))
 1076 			goto isolate_fail;
 1077 
 1078 		/*
 1079 		 * Migration will fail if an anonymous page is pinned in memory,
 1080 		 * so avoid taking lru_lock and isolating it unnecessarily in an
 1081 		 * admittedly racy check.
 1082 		 */
 1083 		mapping = folio_mapping(folio);
 1084 		if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
 1085 			goto isolate_fail_put;
 1086 
 1087 		/*
 1088 		 * Only allow to migrate anonymous pages in GFP_NOFS context
 1089 		 * because those do not depend on fs locks.
 1090 		 */
 1091 		if (!(cc->gfp_mask & __GFP_FS) && mapping)
 1092 			goto isolate_fail_put;
 1093 
 1094 		/* Only take pages on LRU: a check now makes later tests safe */
 1095 		if (!folio_test_lru(folio))
 1096 			goto isolate_fail_put;
 1097 
 1098 		is_unevictable = folio_test_unevictable(folio);
 1099 
 1100 		/* Compaction might skip unevictable pages but CMA takes them */
 1101 		if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
 1102 			goto isolate_fail_put;
 1103 
 1104 		/*
 1105 		 * To minimise LRU disruption, the caller can indicate with
 1106 		 * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages
 1107 		 * it will be able to migrate without blocking - clean pages
 1108 		 * for the most part.  PageWriteback would require blocking.
 1109 		 */
 1110 		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
 1111 			goto isolate_fail_put;
 1112 
 1113 		is_dirty = folio_test_dirty(folio);
 1114 
 1115 		if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||
 1116 		    (mapping && is_unevictable)) {
 1117 			bool migrate_dirty = true;
 1118 			bool is_inaccessible;
 1119 
 1120 			/*
 1121 			 * Only folios without mappings or that have
 1122 			 * a ->migrate_folio callback are possible to migrate
 1123 			 * without blocking.
 1124 			 *
 1125 			 * Folios from inaccessible mappings are not migratable.
 1126 			 *
 1127 			 * However, we can be racing with truncation, which can
 1128 			 * free the mapping that we need to check. Truncation
 1129 			 * holds the folio lock until after the folio is removed
 1130 			 * from the page so holding it ourselves is sufficient.
 1131 			 *
 1132 			 * To avoid locking the folio just to check inaccessible,
 1133 			 * assume every inaccessible folio is also unevictable,
 1134 			 * which is a cheaper test.  If our assumption goes
 1135 			 * wrong, it's not a correctness bug, just potentially
 1136 			 * wasted cycles.
 1137 			 */
 1138 			if (!folio_trylock(folio))
 1139 				goto isolate_fail_put;
 1140 
 1141 			mapping = folio_mapping(folio);
 1142 			if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {
 1143 				migrate_dirty = !mapping ||
 1144 						mapping->a_ops->migrate_folio;
 1145 			}
 1146 			is_inaccessible = mapping && mapping_inaccessible(mapping);
 1147 			folio_unlock(folio);
 1148 			if (!migrate_dirty || is_inaccessible)
 1149 				goto isolate_fail_put;
 1150 		}
 1151 
 1152 		/* Try isolate the folio */
 1153 		if (!folio_test_clear_lru(folio))
 1154 			goto isolate_fail_put;
 1155 
 1156 		lruvec = folio_lruvec(folio);
 1157 
 1158 		/* If we already hold the lock, we can skip some rechecking */
 1159 		if (lruvec != locked) {
 1160 			if (locked)
 1161 				unlock_page_lruvec_irqrestore(locked, flags);
 1162 
 1163 			compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
 1164 			locked = lruvec;
 1165 
 1166 			lruvec_memcg_debug(lruvec, folio);
 1167 
 1168 			/*
 1169 			 * Try get exclusive access under lock. If marked for
 1170 			 * skip, the scan is aborted unless the current context
 1171 			 * is a rescan to reach the end of the pageblock.
 1172 			 */
 1173 			if (!skip_updated && valid_page) {
 1174 				skip_updated = true;
 1175 				if (test_and_set_skip(cc, valid_page) &&
 1176 				    !cc->finish_pageblock) {
 1177 					low_pfn = end_pfn;
 1178 					goto isolate_abort;
 1179 				}
 1180 			}
 1181 
 1182 			/*
 1183 			 * Check LRU folio order under the lock
 1184 			 */
 1185 			if (unlikely(skip_isolation_on_order(folio_order(folio),
 1186 							     cc->order) &&
 1187 				     !cc->alloc_contig)) {
 1188 				low_pfn += folio_nr_pages(folio) - 1;
 1189 				nr_scanned += folio_nr_pages(folio) - 1;
 1190 				folio_set_lru(folio);
 1191 				goto isolate_fail_put;
 1192 			}
 1193 		}
 1194 
 1195 		/* The folio is taken off the LRU */
 1196 		if (folio_test_large(folio))
 1197 			low_pfn += folio_nr_pages(folio) - 1;
 1198 
 1199 		/* Successfully isolated */
 1200 		lruvec_del_folio(lruvec, folio);
 1201 		node_stat_mod_folio(folio,
 1202 				NR_ISOLATED_ANON + folio_is_file_lru(folio),
 1203 				folio_nr_pages(folio));
 1204 
 1205 isolate_success:
 1206 		list_add(&folio->lru, &cc->migratepages);
 1207 isolate_success_no_list:
 1208 		cc->nr_migratepages += folio_nr_pages(folio);
 1209 		nr_isolated += folio_nr_pages(folio);
 1210 		nr_scanned += folio_nr_pages(folio) - 1;
 1211 
 1212 		/*
 1213 		 * Avoid isolating too much unless this block is being
 1214 		 * fully scanned (e.g. dirty/writeback pages, parallel allocation)
 1215 		 * or a lock is contended. For contention, isolate quickly to
 1216 		 * potentially remove one source of contention.
 1217 		 */
 1218 		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&
 1219 		    !cc->finish_pageblock && !cc->contended) {
 1220 			++low_pfn;
 1221 			break;
 1222 		}
 1223 
 1224 		continue;
 1225 
 1226 isolate_fail_put:
 1227 		/* Avoid potential deadlock in freeing page under lru_lock */
 1228 		if (locked) {
 1229 			unlock_page_lruvec_irqrestore(locked, flags);
 1230 			locked = NULL;
 1231 		}
 1232 		folio_put(folio);
 1233 
 1234 isolate_fail:
 1235 		if (!skip_on_failure && ret != -ENOMEM)
 1236 			continue;
 1237 
 1238 		/*
 1239 		 * We have isolated some pages, but then failed. Release them
 1240 		 * instead of migrating, as we cannot form the cc->order buddy
 1241 		 * page anyway.
 1242 		 */
 1243 		if (nr_isolated) {
 1244 			if (locked) {
 1245 				unlock_page_lruvec_irqrestore(locked, flags);
 1246 				locked = NULL;
 1247 			}
 1248 			putback_movable_pages(&cc->migratepages);
 1249 			cc->nr_migratepages = 0;
 1250 			nr_isolated = 0;
 1251 		}
 1252 
 1253 		if (low_pfn < next_skip_pfn) {
 1254 			low_pfn = next_skip_pfn - 1;
 1255 			/*
 1256 			 * The check near the loop beginning would have updated
 1257 			 * next_skip_pfn too, but this is a bit simpler.
 1258 			 */
 1259 			next_skip_pfn += 1UL << cc->order;
 1260 		}
 1261 
 1262 		if (ret == -ENOMEM)
 1263 			break;
 1264 	}
 1265 
 1266 	/*
 1267 	 * The PageBuddy() check could have potentially brought us outside
 1268 	 * the range to be scanned.
 1269 	 */
 1270 	if (unlikely(low_pfn > end_pfn))
 1271 		low_pfn = end_pfn;
 1272 
 1273 	folio = NULL;
 1274 
 1275 isolate_abort:
 1276 	if (locked)
 1277 		unlock_page_lruvec_irqrestore(locked, flags);
 1278 	if (folio) {
 1279 		folio_set_lru(folio);
 1280 		folio_put(folio);
 1281 	}
 1282 
 1283 	/*
 1284 	 * Update the cached scanner pfn once the pageblock has been scanned.
 1285 	 * Pages will either be migrated in which case there is no point
 1286 	 * scanning in the near future or migration failed in which case the
 1287 	 * failure reason may persist. The block is marked for skipping if
 1288 	 * there were no pages isolated in the block or if the block is
 1289 	 * rescanned twice in a row.
 1290 	 */
 1291 	if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) {
 1292 		if (!cc->no_set_skip_hint && valid_page && !skip_updated)
 1293 			set_pageblock_skip(valid_page);
 1294 		update_cached_migrate(cc, low_pfn);
 1295 	}
 1296 
 1297 	trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
 1298 						nr_scanned, nr_isolated);
 1299 
 1300 fatal_pending:
 1301 	cc->total_migrate_scanned += nr_scanned;
 1302 	if (nr_isolated)
 1303 		count_compact_events(COMPACTISOLATED, nr_isolated);
 1304 
 1305 	cc->migrate_pfn = low_pfn;
 1306 
 1307 	return ret;
 1308 }
 1309 
 1310 /**
 1311  * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
 1312  * @cc:        Compaction control structure.
 1313  * @start_pfn: The first PFN to start isolating.
 1314  * @end_pfn:   The one-past-last PFN.
 1315  *
 1316  * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
 1317  * in case we could not allocate a page, or 0.
 1318  */
 1319 int
 1320 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
 1321 							unsigned long end_pfn)
 1322 {
 1323 	unsigned long pfn, block_start_pfn, block_end_pfn;
 1324 	int ret = 0;
 1325 
 1326 	/* Scan block by block. First and last block may be incomplete */
 1327 	pfn = start_pfn;
 1328 	block_start_pfn = pageblock_start_pfn(pfn);
 1329 	if (block_start_pfn < cc->zone->zone_start_pfn)
 1330 		block_start_pfn = cc->zone->zone_start_pfn;
 1331 	block_end_pfn = pageblock_end_pfn(pfn);
 1332 
 1333 	for (; pfn < end_pfn; pfn = block_end_pfn,
 1334 				block_start_pfn = block_end_pfn,
 1335 				block_end_pfn += pageblock_nr_pages) {
 1336 
 1337 		block_end_pfn = min(block_end_pfn, end_pfn);
 1338 
 1339 		if (!pageblock_pfn_to_page(block_start_pfn,
 1340 					block_end_pfn, cc->zone))
 1341 			continue;
 1342 
 1343 		ret = isolate_migratepages_block(cc, pfn, block_end_pfn,
 1344 						 ISOLATE_UNEVICTABLE);
 1345 
 1346 		if (ret)
 1347 			break;
 1348 
 1349 		if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)
 1350 			break;
 1351 	}
 1352 
 1353 	return ret;
 1354 }
 1355 
 1356 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
 1357 #ifdef CONFIG_COMPACTION
 1358 
 1359 static bool suitable_migration_source(struct compact_control *cc,
 1360 							struct page *page)
 1361 {
 1362 	int block_mt;
 1363 
 1364 	if (pageblock_skip_persistent(page))
 1365 		return false;
 1366 
 1367 	if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
 1368 		return true;
 1369 
 1370 	block_mt = get_pageblock_migratetype(page);
 1371 
 1372 	if (cc->migratetype == MIGRATE_MOVABLE)
 1373 		return is_migrate_movable(block_mt);
 1374 	else
 1375 		return block_mt == cc->migratetype;
 1376 }
 1377 
 1378 /* Returns true if the page is within a block suitable for migration to */
 1379 static bool suitable_migration_target(struct compact_control *cc,
 1380 							struct page *page)
 1381 {
 1382 	/* If the page is a large free page, then disallow migration */
 1383 	if (PageBuddy(page)) {
 1384 		int order = cc->order > 0 ? cc->order : pageblock_order;
 1385 
 1386 		/*
 1387 		 * We are checking page_order without zone->lock taken. But
 1388 		 * the only small danger is that we skip a potentially suitable
 1389 		 * pageblock, so it's not worth to check order for valid range.
 1390 		 */
 1391 		if (buddy_order_unsafe(page) >= order)
 1392 			return false;
 1393 	}
 1394 
 1395 	if (cc->ignore_block_suitable)
 1396 		return true;
 1397 
 1398 	/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
 1399 	if (is_migrate_movable(get_pageblock_migratetype(page)))
 1400 		return true;
 1401 
 1402 	/* Otherwise skip the block */
 1403 	return false;
 1404 }
 1405 
 1406 static inline unsigned int
 1407 freelist_scan_limit(struct compact_control *cc)
 1408 {
 1409 	unsigned short shift = BITS_PER_LONG - 1;
 1410 
 1411 	return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
 1412 }
 1413 
 1414 /*
 1415  * Test whether the free scanner has reached the same or lower pageblock than
 1416  * the migration scanner, and compaction should thus terminate.
 1417  */
 1418 static inline bool compact_scanners_met(struct compact_control *cc)
 1419 {
 1420 	return (cc->free_pfn >> pageblock_order)
 1421 		<= (cc->migrate_pfn >> pageblock_order);
 1422 }
 1423 
 1424 /*
 1425  * Used when scanning for a suitable migration target which scans freelists
 1426  * in reverse. Reorders the list such as the unscanned pages are scanned
 1427  * first on the next iteration of the free scanner
 1428  */
 1429 static void
 1430 move_freelist_head(struct list_head *freelist, struct page *freepage)
 1431 {
 1432 	LIST_HEAD(sublist);
 1433 
 1434 	if (!list_is_first(&freepage->buddy_list, freelist)) {
 1435 		list_cut_before(&sublist, freelist, &freepage->buddy_list);
 1436 		list_splice_tail(&sublist, freelist);
 1437 	}
 1438 }
 1439 
 1440 /*
 1441  * Similar to move_freelist_head except used by the migration scanner
 1442  * when scanning forward. It's possible for these list operations to
 1443  * move against each other if they search the free list exactly in
 1444  * lockstep.
 1445  */
 1446 static void
 1447 move_freelist_tail(struct list_head *freelist, struct page *freepage)
 1448 {
 1449 	LIST_HEAD(sublist);
 1450 
 1451 	if (!list_is_last(&freepage->buddy_list, freelist)) {
 1452 		list_cut_position(&sublist, freelist, &freepage->buddy_list);
 1453 		list_splice_tail(&sublist, freelist);
 1454 	}
 1455 }
 1456 
 1457 static void
 1458 fast_isolate_around(struct compact_control *cc, unsigned long pfn)
 1459 {
 1460 	unsigned long start_pfn, end_pfn;
 1461 	struct page *page;
 1462 
 1463 	/* Do not search around if there are enough pages already */
 1464 	if (cc->nr_freepages >= cc->nr_migratepages)
 1465 		return;
 1466 
 1467 	/* Minimise scanning during async compaction */
 1468 	if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
 1469 		return;
 1470 
 1471 	/* Pageblock boundaries */
 1472 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);
 1473 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));
 1474 
 1475 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);
 1476 	if (!page)
 1477 		return;
 1478 
 1479 	isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);
 1480 
 1481 	/* Skip this pageblock in the future as it's full or nearly full */
 1482 	if (start_pfn == end_pfn && !cc->no_set_skip_hint)
 1483 		set_pageblock_skip(page);
 1484 }
 1485 
 1486 /* Search orders in round-robin fashion */
 1487 static int next_search_order(struct compact_control *cc, int order)
 1488 {
 1489 	order--;
 1490 	if (order < 0)
 1491 		order = cc->order - 1;
 1492 
 1493 	/* Search wrapped around? */
 1494 	if (order == cc->search_order) {
 1495 		cc->search_order--;
 1496 		if (cc->search_order < 0)
 1497 			cc->search_order = cc->order - 1;
 1498 		return -1;
 1499 	}
 1500 
 1501 	return order;
 1502 }
 1503 
 1504 static void fast_isolate_freepages(struct compact_control *cc)
 1505 {
 1506 	unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1);
 1507 	unsigned int nr_scanned = 0, total_isolated = 0;
 1508 	unsigned long low_pfn, min_pfn, highest = 0;
 1509 	unsigned long nr_isolated = 0;
 1510 	unsigned long distance;
 1511 	struct page *page = NULL;
 1512 	bool scan_start = false;
 1513 	int order;
 1514 
 1515 	/* Full compaction passes in a negative order */
 1516 	if (cc->order <= 0)
 1517 		return;
 1518 
 1519 	/*
 1520 	 * If starting the scan, use a deeper search and use the highest
 1521 	 * PFN found if a suitable one is not found.
 1522 	 */
 1523 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
 1524 		limit = pageblock_nr_pages >> 1;
 1525 		scan_start = true;
 1526 	}
 1527 
 1528 	/*
 1529 	 * Preferred point is in the top quarter of the scan space but take
 1530 	 * a pfn from the top half if the search is problematic.
 1531 	 */
 1532 	distance = (cc->free_pfn - cc->migrate_pfn);
 1533 	low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
 1534 	min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
 1535 
 1536 	if (WARN_ON_ONCE(min_pfn > low_pfn))
 1537 		low_pfn = min_pfn;
 1538 
 1539 	/*
 1540 	 * Search starts from the last successful isolation order or the next
 1541 	 * order to search after a previous failure
 1542 	 */
 1543 	cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
 1544 
 1545 	for (order = cc->search_order;
 1546 	     !page && order >= 0;
 1547 	     order = next_search_order(cc, order)) {
 1548 		struct free_area *area = &cc->zone->free_area[order];
 1549 		struct list_head *freelist;
 1550 		struct page *freepage;
 1551 		unsigned long flags;
 1552 		unsigned int order_scanned = 0;
 1553 		unsigned long high_pfn = 0;
 1554 
 1555 		if (!area->nr_free)
 1556 			continue;
 1557 
 1558 		spin_lock_irqsave(&cc->zone->lock, flags);
 1559 		freelist = &area->free_list[MIGRATE_MOVABLE];
 1560 		list_for_each_entry_reverse(freepage, freelist, buddy_list) {
 1561 			unsigned long pfn;
 1562 
 1563 			order_scanned++;
 1564 			nr_scanned++;
 1565 			pfn = page_to_pfn(freepage);
 1566 
 1567 			if (pfn >= highest)
 1568 				highest = max(pageblock_start_pfn(pfn),
 1569 					      cc->zone->zone_start_pfn);
 1570 
 1571 			if (pfn >= low_pfn) {
 1572 				cc->fast_search_fail = 0;
 1573 				cc->search_order = order;
 1574 				page = freepage;
 1575 				break;
 1576 			}
 1577 
 1578 			if (pfn >= min_pfn && pfn > high_pfn) {
 1579 				high_pfn = pfn;
 1580 
 1581 				/* Shorten the scan if a candidate is found */
 1582 				limit >>= 1;
 1583 			}
 1584 
 1585 			if (order_scanned >= limit)
 1586 				break;
 1587 		}
 1588 
 1589 		/* Use a maximum candidate pfn if a preferred one was not found */
 1590 		if (!page && high_pfn) {
 1591 			page = pfn_to_page(high_pfn);
 1592 
 1593 			/* Update freepage for the list reorder below */
 1594 			freepage = page;
 1595 		}
 1596 
 1597 		/* Reorder to so a future search skips recent pages */
 1598 		move_freelist_head(freelist, freepage);
 1599 
 1600 		/* Isolate the page if available */
 1601 		if (page) {
 1602 			if (__isolate_free_page(page, order)) {
 1603 				set_page_private(page, order);
 1604 				nr_isolated = 1 << order;
 1605 				nr_scanned += nr_isolated - 1;
 1606 				total_isolated += nr_isolated;
 1607 				cc->nr_freepages += nr_isolated;
 1608 				list_add_tail(&page->lru, &cc->freepages[order]);
 1609 				count_compact_events(COMPACTISOLATED, nr_isolated);
 1610 			} else {
 1611 				/* If isolation fails, abort the search */
 1612 				order = cc->search_order + 1;
 1613 				page = NULL;
 1614 			}
 1615 		}
 1616 
 1617 		spin_unlock_irqrestore(&cc->zone->lock, flags);
 1618 
 1619 		/* Skip fast search if enough freepages isolated */
 1620 		if (cc->nr_freepages >= cc->nr_migratepages)
 1621 			break;
 1622 
 1623 		/*
 1624 		 * Smaller scan on next order so the total scan is related
 1625 		 * to freelist_scan_limit.
 1626 		 */
 1627 		if (order_scanned >= limit)
 1628 			limit = max(1U, limit >> 1);
 1629 	}
 1630 
 1631 	trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn,
 1632 						   nr_scanned, total_isolated);
 1633 
 1634 	if (!page) {
 1635 		cc->fast_search_fail++;
 1636 		if (scan_start) {
 1637 			/*
 1638 			 * Use the highest PFN found above min. If one was
 1639 			 * not found, be pessimistic for direct compaction
 1640 			 * and use the min mark.
 1641 			 */
 1642 			if (highest >= min_pfn) {
 1643 				page = pfn_to_page(highest);
 1644 				cc->free_pfn = highest;
 1645 			} else {
 1646 				if (cc->direct_compaction && pfn_valid(min_pfn)) {
 1647 					page = pageblock_pfn_to_page(min_pfn,
 1648 						min(pageblock_end_pfn(min_pfn),
 1649 						    zone_end_pfn(cc->zone)),
 1650 						cc->zone);
 1651 					if (page && !suitable_migration_target(cc, page))
 1652 						page = NULL;
 1653 
 1654 					cc->free_pfn = min_pfn;
 1655 				}
 1656 			}
 1657 		}
 1658 	}
 1659 
 1660 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {
 1661 		highest -= pageblock_nr_pages;
 1662 		cc->zone->compact_cached_free_pfn = highest;
 1663 	}
 1664 
 1665 	cc->total_free_scanned += nr_scanned;
 1666 	if (!page)
 1667 		return;
 1668 
 1669 	low_pfn = page_to_pfn(page);
 1670 	fast_isolate_around(cc, low_pfn);
 1671 }
 1672 
 1673 /*
 1674  * Based on information in the current compact_control, find blocks
 1675  * suitable for isolating free pages from and then isolate them.
 1676  */
 1677 static void isolate_freepages(struct compact_control *cc)
 1678 {
 1679 	struct zone *zone = cc->zone;
 1680 	struct page *page;
 1681 	unsigned long block_start_pfn;	/* start of current pageblock */
 1682 	unsigned long isolate_start_pfn; /* exact pfn we start at */
 1683 	unsigned long block_end_pfn;	/* end of current pageblock */
 1684 	unsigned long low_pfn;	     /* lowest pfn scanner is able to scan */
 1685 	unsigned int stride;
 1686 
 1687 	/* Try a small search of the free lists for a candidate */
 1688 	fast_isolate_freepages(cc);
 1689 	if (cc->nr_freepages)
 1690 		return;
 1691 
 1692 	/*
 1693 	 * Initialise the free scanner. The starting point is where we last
 1694 	 * successfully isolated from, zone-cached value, or the end of the
 1695 	 * zone when isolating for the first time. For looping we also need
 1696 	 * this pfn aligned down to the pageblock boundary, because we do
 1697 	 * block_start_pfn -= pageblock_nr_pages in the for loop.
 1698 	 * For ending point, take care when isolating in last pageblock of a
 1699 	 * zone which ends in the middle of a pageblock.
 1700 	 * The low boundary is the end of the pageblock the migration scanner
 1701 	 * is using.
 1702 	 */
 1703 	isolate_start_pfn = cc->free_pfn;
 1704 	block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
 1705 	block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
 1706 						zone_end_pfn(zone));
 1707 	low_pfn = pageblock_end_pfn(cc->migrate_pfn);
 1708 	stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
 1709 
 1710 	/*
 1711 	 * Isolate free pages until enough are available to migrate the
 1712 	 * pages on cc->migratepages. We stop searching if the migrate
 1713 	 * and free page scanners meet or enough free pages are isolated.
 1714 	 */
 1715 	for (; block_start_pfn >= low_pfn;
 1716 				block_end_pfn = block_start_pfn,
 1717 				block_start_pfn -= pageblock_nr_pages,
 1718 				isolate_start_pfn = block_start_pfn) {
 1719 		unsigned long nr_isolated;
 1720 
 1721 		/*
 1722 		 * This can iterate a massively long zone without finding any
 1723 		 * suitable migration targets, so periodically check resched.
 1724 		 */
 1725 		if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
 1726 			cond_resched();
 1727 
 1728 		page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
 1729 									zone);
 1730 		if (!page) {
 1731 			unsigned long next_pfn;
 1732 
 1733 			next_pfn = skip_offline_sections_reverse(block_start_pfn);
 1734 			if (next_pfn)
 1735 				block_start_pfn = max(next_pfn, low_pfn);
 1736 
 1737 			continue;
 1738 		}
 1739 
 1740 		/* Check the block is suitable for migration */
 1741 		if (!suitable_migration_target(cc, page))
 1742 			continue;
 1743 
 1744 		/* If isolation recently failed, do not retry */
 1745 		if (!isolation_suitable(cc, page))
 1746 			continue;
 1747 
 1748 		/* Found a block suitable for isolating free pages from. */
 1749 		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
 1750 					block_end_pfn, cc->freepages, stride, false);
 1751 
 1752 		/* Update the skip hint if the full pageblock was scanned */
 1753 		if (isolate_start_pfn == block_end_pfn)
 1754 			update_pageblock_skip(cc, page, block_start_pfn -
 1755 					      pageblock_nr_pages);
 1756 
 1757 		/* Are enough freepages isolated? */
 1758 		if (cc->nr_freepages >= cc->nr_migratepages) {
 1759 			if (isolate_start_pfn >= block_end_pfn) {
 1760 				/*
 1761 				 * Restart at previous pageblock if more
 1762 				 * freepages can be isolated next time.
 1763 				 */
 1764 				isolate_start_pfn =
 1765 					block_start_pfn - pageblock_nr_pages;
 1766 			}
 1767 			break;
 1768 		} else if (isolate_start_pfn < block_end_pfn) {
 1769 			/*
 1770 			 * If isolation failed early, do not continue
 1771 			 * needlessly.
 1772 			 */
 1773 			break;
 1774 		}
 1775 
 1776 		/* Adjust stride depending on isolation */
 1777 		if (nr_isolated) {
 1778 			stride = 1;
 1779 			continue;
 1780 		}
 1781 		stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
 1782 	}
 1783 
 1784 	/*
 1785 	 * Record where the free scanner will restart next time. Either we
 1786 	 * broke from the loop and set isolate_start_pfn based on the last
 1787 	 * call to isolate_freepages_block(), or we met the migration scanner
 1788 	 * and the loop terminated due to isolate_start_pfn < low_pfn
 1789 	 */
 1790 	cc->free_pfn = isolate_start_pfn;
 1791 }
 1792 
 1793 /*
 1794  * This is a migrate-callback that "allocates" freepages by taking pages
 1795  * from the isolated freelists in the block we are migrating to.
 1796  */
 1797 static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)
 1798 {
 1799 	struct compact_control *cc = (struct compact_control *)data;
 1800 	struct folio *dst;
 1801 	int order = folio_order(src);
 1802 	bool has_isolated_pages = false;
 1803 	int start_order;
 1804 	struct page *freepage;
 1805 	unsigned long size;
 1806 
 1807 again:
 1808 	for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)
 1809 		if (!list_empty(&cc->freepages[start_order]))
 1810 			break;
 1811 
 1812 	/* no free pages in the list */
 1813 	if (start_order == NR_PAGE_ORDERS) {
 1814 		if (has_isolated_pages)
 1815 			return NULL;
 1816 		isolate_freepages(cc);
 1817 		has_isolated_pages = true;
 1818 		goto again;
 1819 	}
 1820 
 1821 	freepage = list_first_entry(&cc->freepages[start_order], struct page,
 1822 				lru);
 1823 	size = 1 << start_order;
 1824 
 1825 	list_del(&freepage->lru);
 1826 
 1827 	while (start_order > order) {
 1828 		start_order--;
 1829 		size >>= 1;
 1830 
 1831 		list_add(&freepage[size].lru, &cc->freepages[start_order]);
 1832 		set_page_private(&freepage[size], start_order);
 1833 	}
 1834 	dst = (struct folio *)freepage;
 1835 
 1836 	post_alloc_hook(&dst->page, order, __GFP_MOVABLE);
 1837 	set_page_refcounted(&dst->page);
 1838 	if (order)
 1839 		prep_compound_page(&dst->page, order);
 1840 	cc->nr_freepages -= 1 << order;
 1841 	cc->nr_migratepages -= 1 << order;
 1842 	return page_rmappable_folio(&dst->page);
 1843 }
 1844 
 1845 static struct folio *compaction_alloc(struct folio *src, unsigned long data)
 1846 {
 1847 	return alloc_hooks(compaction_alloc_noprof(src, data));
 1848 }
 1849 
 1850 /*
 1851  * This is a migrate-callback that "frees" freepages back to the isolated
 1852  * freelist.  All pages on the freelist are from the same zone, so there is no
 1853  * special handling needed for NUMA.
 1854  */
 1855 static void compaction_free(struct folio *dst, unsigned long data)
 1856 {
 1857 	struct compact_control *cc = (struct compact_control *)data;
 1858 	int order = folio_order(dst);
 1859 	struct page *page = &dst->page;
 1860 
 1861 	if (folio_put_testzero(dst)) {
 1862 		free_pages_prepare(page, order);
 1863 		list_add(&dst->lru, &cc->freepages[order]);
 1864 		cc->nr_freepages += 1 << order;
 1865 	}
 1866 	cc->nr_migratepages += 1 << order;
 1867 	/*
 1868 	 * someone else has referenced the page, we cannot take it back to our
 1869 	 * free list.
 1870 	 */
 1871 }
 1872 
 1873 /* possible outcome of isolate_migratepages */
 1874 typedef enum {
 1875 	ISOLATE_ABORT,		/* Abort compaction now */
 1876 	ISOLATE_NONE,		/* No pages isolated, continue scanning */
 1877 	ISOLATE_SUCCESS,	/* Pages isolated, migrate */
 1878 } isolate_migrate_t;
 1879 
 1880 /*
 1881  * Allow userspace to control policy on scanning the unevictable LRU for
 1882  * compactable pages.
 1883  */
 1884 static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
 1885 /*
 1886  * Tunable for proactive compaction. It determines how
 1887  * aggressively the kernel should compact memory in the
 1888  * background. It takes values in the range [0, 100].
 1889  */
 1890 static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
 1891 static int sysctl_extfrag_threshold = 500;
 1892 static int __read_mostly sysctl_compact_memory;
 1893 
 1894 static inline void
 1895 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
 1896 {
 1897 	if (cc->fast_start_pfn == ULONG_MAX)
 1898 		return;
 1899 
 1900 	if (!cc->fast_start_pfn)
 1901 		cc->fast_start_pfn = pfn;
 1902 
 1903 	cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
 1904 }
 1905 
 1906 static inline unsigned long
 1907 reinit_migrate_pfn(struct compact_control *cc)
 1908 {
 1909 	if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
 1910 		return cc->migrate_pfn;
 1911 
 1912 	cc->migrate_pfn = cc->fast_start_pfn;
 1913 	cc->fast_start_pfn = ULONG_MAX;
 1914 
 1915 	return cc->migrate_pfn;
 1916 }
 1917 
 1918 /*
 1919  * Briefly search the free lists for a migration source that already has
 1920  * some free pages to reduce the number of pages that need migration
 1921  * before a pageblock is free.
 1922  */
 1923 static unsigned long fast_find_migrateblock(struct compact_control *cc)
 1924 {
 1925 	unsigned int limit = freelist_scan_limit(cc);
 1926 	unsigned int nr_scanned = 0;
 1927 	unsigned long distance;
 1928 	unsigned long pfn = cc->migrate_pfn;
 1929 	unsigned long high_pfn;
 1930 	int order;
 1931 	bool found_block = false;
 1932 
 1933 	/* Skip hints are relied on to avoid repeats on the fast search */
 1934 	if (cc->ignore_skip_hint)
 1935 		return pfn;
 1936 
 1937 	/*
 1938 	 * If the pageblock should be finished then do not select a different
 1939 	 * pageblock.
 1940 	 */
 1941 	if (cc->finish_pageblock)
 1942 		return pfn;
 1943 
 1944 	/*
 1945 	 * If the migrate_pfn is not at the start of a zone or the start
 1946 	 * of a pageblock then assume this is a continuation of a previous
 1947 	 * scan restarted due to COMPACT_CLUSTER_MAX.
 1948 	 */
 1949 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
 1950 		return pfn;
 1951 
 1952 	/*
 1953 	 * For smaller orders, just linearly scan as the number of pages
 1954 	 * to migrate should be relatively small and does not necessarily
 1955 	 * justify freeing up a large block for a small allocation.
 1956 	 */
 1957 	if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
 1958 		return pfn;
 1959 
 1960 	/*
 1961 	 * Only allow kcompactd and direct requests for movable pages to
 1962 	 * quickly clear out a MOVABLE pageblock for allocation. This
 1963 	 * reduces the risk that a large movable pageblock is freed for
 1964 	 * an unmovable/reclaimable small allocation.
 1965 	 */
 1966 	if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
 1967 		return pfn;
 1968 
 1969 	/*
 1970 	 * When starting the migration scanner, pick any pageblock within the
 1971 	 * first half of the search space. Otherwise try and pick a pageblock
 1972 	 * within the first eighth to reduce the chances that a migration
 1973 	 * target later becomes a source.
 1974 	 */
 1975 	distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
 1976 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)
 1977 		distance >>= 2;
 1978 	high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
 1979 
 1980 	for (order = cc->order - 1;
 1981 	     order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;
 1982 	     order--) {
 1983 		struct free_area *area = &cc->zone->free_area[order];
 1984 		struct list_head *freelist;
 1985 		unsigned long flags;
 1986 		struct page *freepage;
 1987 
 1988 		if (!area->nr_free)
 1989 			continue;
 1990 
 1991 		spin_lock_irqsave(&cc->zone->lock, flags);
 1992 		freelist = &area->free_list[MIGRATE_MOVABLE];
 1993 		list_for_each_entry(freepage, freelist, buddy_list) {
 1994 			unsigned long free_pfn;
 1995 
 1996 			if (nr_scanned++ >= limit) {
 1997 				move_freelist_tail(freelist, freepage);
 1998 				break;
 1999 			}
 2000 
 2001 			free_pfn = page_to_pfn(freepage);
 2002 			if (free_pfn < high_pfn) {
 2003 				/*
 2004 				 * Avoid if skipped recently. Ideally it would
 2005 				 * move to the tail but even safe iteration of
 2006 				 * the list assumes an entry is deleted, not
 2007 				 * reordered.
 2008 				 */
 2009 				if (get_pageblock_skip(freepage))
 2010 					continue;
 2011 
 2012 				/* Reorder to so a future search skips recent pages */
 2013 				move_freelist_tail(freelist, freepage);
 2014 
 2015 				update_fast_start_pfn(cc, free_pfn);
 2016 				pfn = pageblock_start_pfn(free_pfn);
 2017 				if (pfn < cc->zone->zone_start_pfn)
 2018 					pfn = cc->zone->zone_start_pfn;
 2019 				cc->fast_search_fail = 0;
 2020 				found_block = true;
 2021 				break;
 2022 			}
 2023 		}
 2024 		spin_unlock_irqrestore(&cc->zone->lock, flags);
 2025 	}
 2026 
 2027 	cc->total_migrate_scanned += nr_scanned;
 2028 
 2029 	/*
 2030 	 * If fast scanning failed then use a cached entry for a page block
 2031 	 * that had free pages as the basis for starting a linear scan.
 2032 	 */
 2033 	if (!found_block) {
 2034 		cc->fast_search_fail++;
 2035 		pfn = reinit_migrate_pfn(cc);
 2036 	}
 2037 	return pfn;
 2038 }
 2039 
 2040 /*
 2041  * Isolate all pages that can be migrated from the first suitable block,
 2042  * starting at the block pointed to by the migrate scanner pfn within
 2043  * compact_control.
 2044  */
 2045 static isolate_migrate_t isolate_migratepages(struct compact_control *cc)
 2046 {
 2047 	unsigned long block_start_pfn;
 2048 	unsigned long block_end_pfn;
 2049 	unsigned long low_pfn;
 2050 	struct page *page;
 2051 	const isolate_mode_t isolate_mode =
 2052 		(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
 2053 		(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
 2054 	bool fast_find_block;
 2055 
 2056 	/*
 2057 	 * Start at where we last stopped, or beginning of the zone as
 2058 	 * initialized by compact_zone(). The first failure will use
 2059 	 * the lowest PFN as the starting point for linear scanning.
 2060 	 */
 2061 	low_pfn = fast_find_migrateblock(cc);
 2062 	block_start_pfn = pageblock_start_pfn(low_pfn);
 2063 	if (block_start_pfn < cc->zone->zone_start_pfn)
 2064 		block_start_pfn = cc->zone->zone_start_pfn;
 2065 
 2066 	/*
 2067 	 * fast_find_migrateblock() has already ensured the pageblock is not
 2068 	 * set with a skipped flag, so to avoid the isolation_suitable check
 2069 	 * below again, check whether the fast search was successful.
 2070 	 */
 2071 	fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
 2072 
 2073 	/* Only scan within a pageblock boundary */
 2074 	block_end_pfn = pageblock_end_pfn(low_pfn);
 2075 
 2076 	/*
 2077 	 * Iterate over whole pageblocks until we find the first suitable.
 2078 	 * Do not cross the free scanner.
 2079 	 */
 2080 	for (; block_end_pfn <= cc->free_pfn;
 2081 			fast_find_block = false,
 2082 			cc->migrate_pfn = low_pfn = block_end_pfn,
 2083 			block_start_pfn = block_end_pfn,
 2084 			block_end_pfn += pageblock_nr_pages) {
 2085 
 2086 		/*
 2087 		 * This can potentially iterate a massively long zone with
 2088 		 * many pageblocks unsuitable, so periodically check if we
 2089 		 * need to schedule.
 2090 		 */
 2091 		if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
 2092 			cond_resched();
 2093 
 2094 		page = pageblock_pfn_to_page(block_start_pfn,
 2095 						block_end_pfn, cc->zone);
 2096 		if (!page) {
 2097 			unsigned long next_pfn;
 2098 
 2099 			next_pfn = skip_offline_sections(block_start_pfn);
 2100 			if (next_pfn)
 2101 				block_end_pfn = min(next_pfn, cc->free_pfn);
 2102 			continue;
 2103 		}
 2104 
 2105 		/*
 2106 		 * If isolation recently failed, do not retry. Only check the
 2107 		 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
 2108 		 * to be visited multiple times. Assume skip was checked
 2109 		 * before making it "skip" so other compaction instances do
 2110 		 * not scan the same block.
 2111 		 */
 2112 		if ((pageblock_aligned(low_pfn) ||
 2113 		     low_pfn == cc->zone->zone_start_pfn) &&
 2114 		    !fast_find_block && !isolation_suitable(cc, page))
 2115 			continue;
 2116 
 2117 		/*
 2118 		 * For async direct compaction, only scan the pageblocks of the
 2119 		 * same migratetype without huge pages. Async direct compaction
 2120 		 * is optimistic to see if the minimum amount of work satisfies
 2121 		 * the allocation. The cached PFN is updated as it's possible
 2122 		 * that all remaining blocks between source and target are
 2123 		 * unsuitable and the compaction scanners fail to meet.
 2124 		 */
 2125 		if (!suitable_migration_source(cc, page)) {
 2126 			update_cached_migrate(cc, block_end_pfn);
 2127 			continue;
 2128 		}
 2129 
 2130 		/* Perform the isolation */
 2131 		if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,
 2132 						isolate_mode))
 2133 			return ISOLATE_ABORT;
 2134 
 2135 		/*
 2136 		 * Either we isolated something and proceed with migration. Or
 2137 		 * we failed and compact_zone should decide if we should
 2138 		 * continue or not.
 2139 		 */
 2140 		break;
 2141 	}
 2142 
 2143 	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
 2144 }
 2145 
 2146 /*
 2147  * Determine whether kswapd is (or recently was!) running on this node.
 2148  *
 2149  * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't
 2150  * zero it.
 2151  */
 2152 static bool kswapd_is_running(pg_data_t *pgdat)
 2153 {
 2154 	bool running;
 2155 
 2156 	pgdat_kswapd_lock(pgdat);
 2157 	running = pgdat->kswapd && task_is_running(pgdat->kswapd);
 2158 	pgdat_kswapd_unlock(pgdat);
 2159 
 2160 	return running;
 2161 }
 2162 
 2163 /*
 2164  * A zone's fragmentation score is the external fragmentation wrt to the
 2165  * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].
 2166  */
 2167 static unsigned int fragmentation_score_zone(struct zone *zone)
 2168 {
 2169 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);
 2170 }
 2171 
 2172 /*
 2173  * A weighted zone's fragmentation score is the external fragmentation
 2174  * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It
 2175  * returns a value in the range [0, 100].
 2176  *
 2177  * The scaling factor ensures that proactive compaction focuses on larger
 2178  * zones like ZONE_NORMAL, rather than smaller, specialized zones like
 2179  * ZONE_DMA32. For smaller zones, the score value remains close to zero,
 2180  * and thus never exceeds the high threshold for proactive compaction.
 2181  */
 2182 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)
 2183 {
 2184 	unsigned long score;
 2185 
 2186 	score = zone->present_pages * fragmentation_score_zone(zone);
 2187 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);
 2188 }
 2189 
 2190 /*
 2191  * The per-node proactive (background) compaction process is started by its
 2192  * corresponding kcompactd thread when the node's fragmentation score
 2193  * exceeds the high threshold. The compaction process remains active till
 2194  * the node's score falls below the low threshold, or one of the back-off
 2195  * conditions is met.
 2196  */
 2197 static unsigned int fragmentation_score_node(pg_data_t *pgdat)
 2198 {
 2199 	unsigned int score = 0;
 2200 	int zoneid;
 2201 
 2202 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
 2203 		struct zone *zone;
 2204 
 2205 		zone = &pgdat->node_zones[zoneid];
 2206 		if (!populated_zone(zone))
 2207 			continue;
 2208 		score += fragmentation_score_zone_weighted(zone);
 2209 	}
 2210 
 2211 	return score;
 2212 }
 2213 
 2214 static unsigned int fragmentation_score_wmark(bool low)
 2215 {
 2216 	unsigned int wmark_low, leeway;
 2217 
 2218 	wmark_low = 100U - sysctl_compaction_proactiveness;
 2219 	leeway = min(10U, wmark_low / 2);
 2220 	return low ? wmark_low : min(wmark_low + leeway, 100U);
 2221 }
 2222 
 2223 static bool should_proactive_compact_node(pg_data_t *pgdat)
 2224 {
 2225 	int wmark_high;
 2226 
 2227 	if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))
 2228 		return false;
 2229 
 2230 	wmark_high = fragmentation_score_wmark(false);
 2231 	return fragmentation_score_node(pgdat) > wmark_high;
 2232 }
 2233 
 2234 static enum compact_result __compact_finished(struct compact_control *cc)
 2235 {
 2236 	unsigned int order;
 2237 	const int migratetype = cc->migratetype;
 2238 	int ret;
 2239 
 2240 	/* Compaction run completes if the migrate and free scanner meet */
 2241 	if (compact_scanners_met(cc)) {
 2242 		/* Let the next compaction start anew. */
 2243 		reset_cached_positions(cc->zone);
 2244 
 2245 		/*
 2246 		 * Mark that the PG_migrate_skip information should be cleared
 2247 		 * by kswapd when it goes to sleep. kcompactd does not set the
 2248 		 * flag itself as the decision to be clear should be directly
 2249 		 * based on an allocation request.
 2250 		 */
 2251 		if (cc->direct_compaction)
 2252 			cc->zone->compact_blockskip_flush = true;
 2253 
 2254 		if (cc->whole_zone)
 2255 			return COMPACT_COMPLETE;
 2256 		else
 2257 			return COMPACT_PARTIAL_SKIPPED;
 2258 	}
 2259 
 2260 	if (cc->proactive_compaction) {
 2261 		int score, wmark_low;
 2262 		pg_data_t *pgdat;
 2263 
 2264 		pgdat = cc->zone->zone_pgdat;
 2265 		if (kswapd_is_running(pgdat))
 2266 			return COMPACT_PARTIAL_SKIPPED;
 2267 
 2268 		score = fragmentation_score_zone(cc->zone);
 2269 		wmark_low = fragmentation_score_wmark(true);
 2270 
 2271 		if (score > wmark_low)
 2272 			ret = COMPACT_CONTINUE;
 2273 		else
 2274 			ret = COMPACT_SUCCESS;
 2275 
 2276 		goto out;
 2277 	}
 2278 
 2279 	if (is_via_compact_memory(cc->order))
 2280 		return COMPACT_CONTINUE;
 2281 
 2282 	/*
 2283 	 * Always finish scanning a pageblock to reduce the possibility of
 2284 	 * fallbacks in the future. This is particularly important when
 2285 	 * migration source is unmovable/reclaimable but it's not worth
 2286 	 * special casing.
 2287 	 */
 2288 	if (!pageblock_aligned(cc->migrate_pfn))
 2289 		return COMPACT_CONTINUE;
 2290 
 2291 	/*
 2292 	 * When defrag_mode is enabled, make kcompactd target
 2293 	 * watermarks in whole pageblocks. Because they can be stolen
 2294 	 * without polluting, no further fallback checks are needed.
 2295 	 */
 2296 	if (defrag_mode && !cc->direct_compaction) {
 2297 		if (__zone_watermark_ok(cc->zone, cc->order,
 2298 					high_wmark_pages(cc->zone),
 2299 					cc->highest_zoneidx, cc->alloc_flags,
 2300 					zone_page_state(cc->zone,
 2301 							NR_FREE_PAGES_BLOCKS)))
 2302 			return COMPACT_SUCCESS;
 2303 
 2304 		return COMPACT_CONTINUE;
 2305 	}
 2306 
 2307 	/* Direct compactor: Is a suitable page free? */
 2308 	ret = COMPACT_NO_SUITABLE_PAGE;
 2309 	for (order = cc->order; order < NR_PAGE_ORDERS; order++) {
 2310 		struct free_area *area = &cc->zone->free_area[order];
 2311 
 2312 		/* Job done if page is free of the right migratetype */
 2313 		if (!free_area_empty(area, migratetype))
 2314 			return COMPACT_SUCCESS;
 2315 
 2316 #ifdef CONFIG_CMA
 2317 		/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
 2318 		if (migratetype == MIGRATE_MOVABLE &&
 2319 			!free_area_empty(area, MIGRATE_CMA))
 2320 			return COMPACT_SUCCESS;
 2321 #endif
 2322 		/*
 2323 		 * Job done if allocation would steal freepages from
 2324 		 * other migratetype buddy lists.
 2325 		 */
 2326 		if (find_suitable_fallback(area, order, migratetype, true) >= 0)
 2327 			/*
 2328 			 * Movable pages are OK in any pageblock. If we are
 2329 			 * stealing for a non-movable allocation, make sure
 2330 			 * we finish compacting the current pageblock first
 2331 			 * (which is assured by the above migrate_pfn align
 2332 			 * check) so it is as free as possible and we won't
 2333 			 * have to steal another one soon.
 2334 			 */
 2335 			return COMPACT_SUCCESS;
 2336 	}
 2337 
 2338 out:
 2339 	if (cc->contended || fatal_signal_pending(current))
 2340 		ret = COMPACT_CONTENDED;
 2341 
 2342 	return ret;
 2343 }
 2344 
 2345 static enum compact_result compact_finished(struct compact_control *cc)
 2346 {
 2347 	int ret;
 2348 
 2349 	ret = __compact_finished(cc);
 2350 	trace_mm_compaction_finished(cc->zone, cc->order, ret);
 2351 	if (ret == COMPACT_NO_SUITABLE_PAGE)
 2352 		ret = COMPACT_CONTINUE;
 2353 
 2354 	return ret;
 2355 }
 2356 
 2357 static bool __compaction_suitable(struct zone *zone, int order,
 2358 				  unsigned long watermark, int highest_zoneidx,
 2359 				  unsigned long free_pages)
 2360 {
 2361 	/*
 2362 	 * Watermarks for order-0 must be met for compaction to be able to
 2363 	 * isolate free pages for migration targets. This means that the
 2364 	 * watermark have to match, or be more pessimistic than the check in
 2365 	 * __isolate_free_page().
 2366 	 *
 2367 	 * For costly orders, we require a higher watermark for compaction to
 2368 	 * proceed to increase its chances.
 2369 	 *
 2370 	 * We use the direct compactor's highest_zoneidx to skip over zones
 2371 	 * where lowmem reserves would prevent allocation even if compaction
 2372 	 * succeeds.
 2373 	 *
 2374 	 * ALLOC_CMA is used, as pages in CMA pageblocks are considered
 2375 	 * suitable migration targets.
 2376 	 */
 2377 	watermark += compact_gap(order);
 2378 	if (order > PAGE_ALLOC_COSTLY_ORDER)
 2379 		watermark += low_wmark_pages(zone) - min_wmark_pages(zone);
 2380 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
 2381 				   ALLOC_CMA, free_pages);
 2382 }
 2383 
 2384 /*
 2385  * compaction_suitable: Is this suitable to run compaction on this zone now?
 2386  */
 2387 bool compaction_suitable(struct zone *zone, int order, unsigned long watermark,
 2388 			 int highest_zoneidx)
 2389 {
 2390 	enum compact_result compact_result;
 2391 	bool suitable;
 2392 
 2393 	suitable = __compaction_suitable(zone, order, watermark, highest_zoneidx,
 2394 					 zone_page_state(zone, NR_FREE_PAGES));
 2395 	/*
 2396 	 * fragmentation index determines if allocation failures are due to
 2397 	 * low memory or external fragmentation
 2398 	 *
 2399 	 * index of -1000 would imply allocations might succeed depending on
 2400 	 * watermarks, but we already failed the high-order watermark check
 2401 	 * index towards 0 implies failure is due to lack of memory
 2402 	 * index towards 1000 implies failure is due to fragmentation
 2403 	 *
 2404 	 * Only compact if a failure would be due to fragmentation. Also
 2405 	 * ignore fragindex for non-costly orders where the alternative to
 2406 	 * a successful reclaim/compaction is OOM. Fragindex and the
 2407 	 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
 2408 	 * excessive compaction for costly orders, but it should not be at the
 2409 	 * expense of system stability.
 2410 	 */
 2411 	if (suitable) {
 2412 		compact_result = COMPACT_CONTINUE;
 2413 		if (order > PAGE_ALLOC_COSTLY_ORDER) {
 2414 			int fragindex = fragmentation_index(zone, order);
 2415 
 2416 			if (fragindex >= 0 &&
 2417 			    fragindex <= sysctl_extfrag_threshold) {
 2418 				suitable = false;
 2419 				compact_result = COMPACT_NOT_SUITABLE_ZONE;
 2420 			}
 2421 		}
 2422 	} else {
 2423 		compact_result = COMPACT_SKIPPED;
 2424 	}
 2425 
 2426 	trace_mm_compaction_suitable(zone, order, compact_result);
 2427 
 2428 	return suitable;
 2429 }
 2430 
 2431 /* Used by direct reclaimers */
 2432 bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
 2433 		int alloc_flags)
 2434 {
 2435 	struct zone *zone;
 2436 	struct zoneref *z;
 2437 
 2438 	/*
 2439 	 * Make sure at least one zone would pass __compaction_suitable if we continue
 2440 	 * retrying the reclaim.
 2441 	 */
 2442 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
 2443 				ac->highest_zoneidx, ac->nodemask) {
 2444 		unsigned long available;
 2445 
 2446 		/*
 2447 		 * Do not consider all the reclaimable memory because we do not
 2448 		 * want to trash just for a single high order allocation which
 2449 		 * is even not guaranteed to appear even if __compaction_suitable
 2450 		 * is happy about the watermark check.
 2451 		 */
 2452 		available = zone_reclaimable_pages(zone) / order;
 2453 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
 2454 		if (__compaction_suitable(zone, order, min_wmark_pages(zone),
 2455 					  ac->highest_zoneidx, available))
 2456 			return true;
 2457 	}
 2458 
 2459 	return false;
 2460 }
 2461 
 2462 /*
 2463  * Should we do compaction for target allocation order.
 2464  * Return COMPACT_SUCCESS if allocation for target order can be already
 2465  * satisfied
 2466  * Return COMPACT_SKIPPED if compaction for target order is likely to fail
 2467  * Return COMPACT_CONTINUE if compaction for target order should be ran
 2468  */
 2469 static enum compact_result
 2470 compaction_suit_allocation_order(struct zone *zone, unsigned int order,
 2471 				 int highest_zoneidx, unsigned int alloc_flags,
 2472 				 bool async, bool kcompactd)
 2473 {
 2474 	unsigned long free_pages;
 2475 	unsigned long watermark;
 2476 
 2477 	if (kcompactd && defrag_mode)
 2478 		free_pages = zone_page_state(zone, NR_FREE_PAGES_BLOCKS);
 2479 	else
 2480 		free_pages = zone_page_state(zone, NR_FREE_PAGES);
 2481 
 2482 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
 2483 	if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx,
 2484 				alloc_flags, free_pages))
 2485 		return COMPACT_SUCCESS;
 2486 
 2487 	/*
 2488 	 * For unmovable allocations (without ALLOC_CMA), check if there is enough
 2489 	 * free memory in the non-CMA pageblocks. Otherwise compaction could form
 2490 	 * the high-order page in CMA pageblocks, which would not help the
 2491 	 * allocation to succeed. However, limit the check to costly order async
 2492 	 * compaction (such as opportunistic THP attempts) because there is the
 2493 	 * possibility that compaction would migrate pages from non-CMA to CMA
 2494 	 * pageblock.
 2495 	 */
 2496 	if (order > PAGE_ALLOC_COSTLY_ORDER && async &&
 2497 	    !(alloc_flags & ALLOC_CMA)) {
 2498 		if (!__zone_watermark_ok(zone, 0, watermark + compact_gap(order),
 2499 					 highest_zoneidx, 0,
 2500 					 zone_page_state(zone, NR_FREE_PAGES)))
 2501 			return COMPACT_SKIPPED;
 2502 	}
 2503 
 2504 	if (!compaction_suitable(zone, order, watermark, highest_zoneidx))
 2505 		return COMPACT_SKIPPED;
 2506 
 2507 	return COMPACT_CONTINUE;
 2508 }
 2509 
 2510 static enum compact_result
 2511 compact_zone(struct compact_control *cc, struct capture_control *capc)
 2512 {
 2513 	enum compact_result ret;
 2514 	unsigned long start_pfn = cc->zone->zone_start_pfn;
 2515 	unsigned long end_pfn = zone_end_pfn(cc->zone);
 2516 	unsigned long last_migrated_pfn;
 2517 	const bool sync = cc->mode != MIGRATE_ASYNC;
 2518 	bool update_cached;
 2519 	unsigned int nr_succeeded = 0, nr_migratepages;
 2520 	int order;
 2521 
 2522 	/*
 2523 	 * These counters track activities during zone compaction.  Initialize
 2524 	 * them before compacting a new zone.
 2525 	 */
 2526 	cc->total_migrate_scanned = 0;
 2527 	cc->total_free_scanned = 0;
 2528 	cc->nr_migratepages = 0;
 2529 	cc->nr_freepages = 0;
 2530 	for (order = 0; order < NR_PAGE_ORDERS; order++)
 2531 		INIT_LIST_HEAD(&cc->freepages[order]);
 2532 	INIT_LIST_HEAD(&cc->migratepages);
 2533 
 2534 	cc->migratetype = gfp_migratetype(cc->gfp_mask);
 2535 
 2536 	if (!is_via_compact_memory(cc->order)) {
 2537 		ret = compaction_suit_allocation_order(cc->zone, cc->order,
 2538 						       cc->highest_zoneidx,
 2539 						       cc->alloc_flags,
 2540 						       cc->mode == MIGRATE_ASYNC,
 2541 						       !cc->direct_compaction);
 2542 		if (ret != COMPACT_CONTINUE)
 2543 			return ret;
 2544 	}
 2545 
 2546 	/*
 2547 	 * Clear pageblock skip if there were failures recently and compaction
 2548 	 * is about to be retried after being deferred.
 2549 	 */
 2550 	if (compaction_restarting(cc->zone, cc->order))
 2551 		__reset_isolation_suitable(cc->zone);
 2552 
 2553 	/*
 2554 	 * Setup to move all movable pages to the end of the zone. Used cached
 2555 	 * information on where the scanners should start (unless we explicitly
 2556 	 * want to compact the whole zone), but check that it is initialised
 2557 	 * by ensuring the values are within zone boundaries.
 2558 	 */
 2559 	cc->fast_start_pfn = 0;
 2560 	if (cc->whole_zone) {
 2561 		cc->migrate_pfn = start_pfn;
 2562 		cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
 2563 	} else {
 2564 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
 2565 		cc->free_pfn = cc->zone->compact_cached_free_pfn;
 2566 		if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
 2567 			cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
 2568 			cc->zone->compact_cached_free_pfn = cc->free_pfn;
 2569 		}
 2570 		if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
 2571 			cc->migrate_pfn = start_pfn;
 2572 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
 2573 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
 2574 		}
 2575 
 2576 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
 2577 			cc->whole_zone = true;
 2578 	}
 2579 
 2580 	last_migrated_pfn = 0;
 2581 
 2582 	/*
 2583 	 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
 2584 	 * the basis that some migrations will fail in ASYNC mode. However,
 2585 	 * if the cached PFNs match and pageblocks are skipped due to having
 2586 	 * no isolation candidates, then the sync state does not matter.
 2587 	 * Until a pageblock with isolation candidates is found, keep the
 2588 	 * cached PFNs in sync to avoid revisiting the same blocks.
 2589 	 */
 2590 	update_cached = !sync &&
 2591 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
 2592 
 2593 	trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);
 2594 
 2595 	/* lru_add_drain_all could be expensive with involving other CPUs */
 2596 	lru_add_drain();
 2597 
 2598 	while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
 2599 		int err;
 2600 		unsigned long iteration_start_pfn = cc->migrate_pfn;
 2601 
 2602 		/*
 2603 		 * Avoid multiple rescans of the same pageblock which can
 2604 		 * happen if a page cannot be isolated (dirty/writeback in
 2605 		 * async mode) or if the migrated pages are being allocated
 2606 		 * before the pageblock is cleared.  The first rescan will
 2607 		 * capture the entire pageblock for migration. If it fails,
 2608 		 * it'll be marked skip and scanning will proceed as normal.
 2609 		 */
 2610 		cc->finish_pageblock = false;
 2611 		if (pageblock_start_pfn(last_migrated_pfn) ==
 2612 		    pageblock_start_pfn(iteration_start_pfn)) {
 2613 			cc->finish_pageblock = true;
 2614 		}
 2615 
 2616 rescan:
 2617 		switch (isolate_migratepages(cc)) {
 2618 		case ISOLATE_ABORT:
 2619 			ret = COMPACT_CONTENDED;
 2620 			putback_movable_pages(&cc->migratepages);
 2621 			cc->nr_migratepages = 0;
 2622 			goto out;
 2623 		case ISOLATE_NONE:
 2624 			if (update_cached) {
 2625 				cc->zone->compact_cached_migrate_pfn[1] =
 2626 					cc->zone->compact_cached_migrate_pfn[0];
 2627 			}
 2628 
 2629 			/*
 2630 			 * We haven't isolated and migrated anything, but
 2631 			 * there might still be unflushed migrations from
 2632 			 * previous cc->order aligned block.
 2633 			 */
 2634 			goto check_drain;
 2635 		case ISOLATE_SUCCESS:
 2636 			update_cached = false;
 2637 			last_migrated_pfn = max(cc->zone->zone_start_pfn,
 2638 				pageblock_start_pfn(cc->migrate_pfn - 1));
 2639 		}
 2640 
 2641 		/*
 2642 		 * Record the number of pages to migrate since the
 2643 		 * compaction_alloc/free() will update cc->nr_migratepages
 2644 		 * properly.
 2645 		 */
 2646 		nr_migratepages = cc->nr_migratepages;
 2647 		err = migrate_pages(&cc->migratepages, compaction_alloc,
 2648 				compaction_free, (unsigned long)cc, cc->mode,
 2649 				MR_COMPACTION, &nr_succeeded);
 2650 
 2651 		trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);
 2652 
 2653 		/* All pages were either migrated or will be released */
 2654 		cc->nr_migratepages = 0;
 2655 		if (err) {
 2656 			putback_movable_pages(&cc->migratepages);
 2657 			/*
 2658 			 * migrate_pages() may return -ENOMEM when scanners meet
 2659 			 * and we want compact_finished() to detect it
 2660 			 */
 2661 			if (err == -ENOMEM && !compact_scanners_met(cc)) {
 2662 				ret = COMPACT_CONTENDED;
 2663 				goto out;
 2664 			}
 2665 			/*
 2666 			 * If an ASYNC or SYNC_LIGHT fails to migrate a page
 2667 			 * within the pageblock_order-aligned block and
 2668 			 * fast_find_migrateblock may be used then scan the
 2669 			 * remainder of the pageblock. This will mark the
 2670 			 * pageblock "skip" to avoid rescanning in the near
 2671 			 * future. This will isolate more pages than necessary
 2672 			 * for the request but avoid loops due to
 2673 			 * fast_find_migrateblock revisiting blocks that were
 2674 			 * recently partially scanned.
 2675 			 */
 2676 			if (!pageblock_aligned(cc->migrate_pfn) &&
 2677 			    !cc->ignore_skip_hint && !cc->finish_pageblock &&
 2678 			    (cc->mode < MIGRATE_SYNC)) {
 2679 				cc->finish_pageblock = true;
 2680 
 2681 				/*
 2682 				 * Draining pcplists does not help THP if
 2683 				 * any page failed to migrate. Even after
 2684 				 * drain, the pageblock will not be free.
 2685 				 */
 2686 				if (cc->order == COMPACTION_HPAGE_ORDER)
 2687 					last_migrated_pfn = 0;
 2688 
 2689 				goto rescan;
 2690 			}
 2691 		}
 2692 
 2693 		/* Stop if a page has been captured */
 2694 		if (capc && capc->page) {
 2695 			ret = COMPACT_SUCCESS;
 2696 			break;
 2697 		}
 2698 
 2699 check_drain:
 2700 		/*
 2701 		 * Has the migration scanner moved away from the previous
 2702 		 * cc->order aligned block where we migrated from? If yes,
 2703 		 * flush the pages that were freed, so that they can merge and
 2704 		 * compact_finished() can detect immediately if allocation
 2705 		 * would succeed.
 2706 		 */
 2707 		if (cc->order > 0 && last_migrated_pfn) {
 2708 			unsigned long current_block_start =
 2709 				block_start_pfn(cc->migrate_pfn, cc->order);
 2710 
 2711 			if (last_migrated_pfn < current_block_start) {
 2712 				lru_add_drain_cpu_zone(cc->zone);
 2713 				/* No more flushing until we migrate again */
 2714 				last_migrated_pfn = 0;
 2715 			}
 2716 		}
 2717 	}
 2718 
 2719 out:
 2720 	/*
 2721 	 * Release free pages and update where the free scanner should restart,
 2722 	 * so we don't leave any returned pages behind in the next attempt.
 2723 	 */
 2724 	if (cc->nr_freepages > 0) {
 2725 		unsigned long free_pfn = release_free_list(cc->freepages);
 2726 
 2727 		cc->nr_freepages = 0;
 2728 		VM_BUG_ON(free_pfn == 0);
 2729 		/* The cached pfn is always the first in a pageblock */
 2730 		free_pfn = pageblock_start_pfn(free_pfn);
 2731 		/*
 2732 		 * Only go back, not forward. The cached pfn might have been
 2733 		 * already reset to zone end in compact_finished()
 2734 		 */
 2735 		if (free_pfn > cc->zone->compact_cached_free_pfn)
 2736 			cc->zone->compact_cached_free_pfn = free_pfn;
 2737 	}
 2738 
 2739 	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
 2740 	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
 2741 
 2742 	trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);
 2743 
 2744 	VM_BUG_ON(!list_empty(&cc->migratepages));
 2745 
 2746 	return ret;
 2747 }
 2748 
 2749 static enum compact_result compact_zone_order(struct zone *zone, int order,
 2750 		gfp_t gfp_mask, enum compact_priority prio,
 2751 		unsigned int alloc_flags, int highest_zoneidx,
 2752 		struct page **capture)
 2753 {
 2754 	enum compact_result ret;
 2755 	struct compact_control cc = {
 2756 		.order = order,
 2757 		.search_order = order,
 2758 		.gfp_mask = gfp_mask,
 2759 		.zone = zone,
 2760 		.mode = (prio == COMPACT_PRIO_ASYNC) ?
 2761 					MIGRATE_ASYNC :	MIGRATE_SYNC_LIGHT,
 2762 		.alloc_flags = alloc_flags,
 2763 		.highest_zoneidx = highest_zoneidx,
 2764 		.direct_compaction = true,
 2765 		.whole_zone = (prio == MIN_COMPACT_PRIORITY),
 2766 		.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
 2767 		.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
 2768 	};
 2769 	struct capture_control capc = {
 2770 		.cc = &cc,
 2771 		.page = NULL,
 2772 	};
 2773 
 2774 	/*
 2775 	 * Make sure the structs are really initialized before we expose the
 2776 	 * capture control, in case we are interrupted and the interrupt handler
 2777 	 * frees a page.
 2778 	 */
 2779 	barrier();
 2780 	WRITE_ONCE(current->capture_control, &capc);
 2781 
 2782 	ret = compact_zone(&cc, &capc);
 2783 
 2784 	/*
 2785 	 * Make sure we hide capture control first before we read the captured
 2786 	 * page pointer, otherwise an interrupt could free and capture a page
 2787 	 * and we would leak it.
 2788 	 */
 2789 	WRITE_ONCE(current->capture_control, NULL);
 2790 	*capture = READ_ONCE(capc.page);
 2791 	/*
 2792 	 * Technically, it is also possible that compaction is skipped but
 2793 	 * the page is still captured out of luck(IRQ came and freed the page).
 2794 	 * Returning COMPACT_SUCCESS in such cases helps in properly accounting
 2795 	 * the COMPACT[STALL|FAIL] when compaction is skipped.
 2796 	 */
 2797 	if (*capture)
 2798 		ret = COMPACT_SUCCESS;
 2799 
 2800 	return ret;
 2801 }
 2802 
 2803 /**
 2804  * try_to_compact_pages - Direct compact to satisfy a high-order allocation
 2805  * @gfp_mask: The GFP mask of the current allocation
 2806  * @order: The order of the current allocation
 2807  * @alloc_flags: The allocation flags of the current allocation
 2808  * @ac: The context of current allocation
 2809  * @prio: Determines how hard direct compaction should try to succeed
 2810  * @capture: Pointer to free page created by compaction will be stored here
 2811  *
 2812  * This is the main entry point for direct page compaction.
 2813  */
 2814 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
 2815 		unsigned int alloc_flags, const struct alloc_context *ac,
 2816 		enum compact_priority prio, struct page **capture)
 2817 {
 2818 	struct zoneref *z;
 2819 	struct zone *zone;
 2820 	enum compact_result rc = COMPACT_SKIPPED;
 2821 
 2822 	if (!gfp_compaction_allowed(gfp_mask))
 2823 		return COMPACT_SKIPPED;
 2824 
 2825 	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
 2826 
 2827 	/* Compact each zone in the list */
 2828 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
 2829 					ac->highest_zoneidx, ac->nodemask) {
 2830 		enum compact_result status;
 2831 
 2832 		if (cpusets_enabled() &&
 2833 			(alloc_flags & ALLOC_CPUSET) &&
 2834 			!__cpuset_zone_allowed(zone, gfp_mask))
 2835 				continue;
 2836 
 2837 		if (prio > MIN_COMPACT_PRIORITY
 2838 					&& compaction_deferred(zone, order)) {
 2839 			rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
 2840 			continue;
 2841 		}
 2842 
 2843 		status = compact_zone_order(zone, order, gfp_mask, prio,
 2844 				alloc_flags, ac->highest_zoneidx, capture);
 2845 		rc = max(status, rc);
 2846 
 2847 		/* The allocation should succeed, stop compacting */
 2848 		if (status == COMPACT_SUCCESS) {
 2849 			/*
 2850 			 * We think the allocation will succeed in this zone,
 2851 			 * but it is not certain, hence the false. The caller
 2852 			 * will repeat this with true if allocation indeed
 2853 			 * succeeds in this zone.
 2854 			 */
 2855 			compaction_defer_reset(zone, order, false);
 2856 
 2857 			break;
 2858 		}
 2859 
 2860 		if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
 2861 					status == COMPACT_PARTIAL_SKIPPED))
 2862 			/*
 2863 			 * We think that allocation won't succeed in this zone
 2864 			 * so we defer compaction there. If it ends up
 2865 			 * succeeding after all, it will be reset.
 2866 			 */
 2867 			defer_compaction(zone, order);
 2868 
 2869 		/*
 2870 		 * We might have stopped compacting due to need_resched() in
 2871 		 * async compaction, or due to a fatal signal detected. In that
 2872 		 * case do not try further zones
 2873 		 */
 2874 		if ((prio == COMPACT_PRIO_ASYNC && need_resched())
 2875 					|| fatal_signal_pending(current))
 2876 			break;
 2877 	}
 2878 
 2879 	return rc;
 2880 }
 2881 
 2882 /*
 2883  * compact_node() - compact all zones within a node
 2884  * @pgdat: The node page data
 2885  * @proactive: Whether the compaction is proactive
 2886  *
 2887  * For proactive compaction, compact till each zone's fragmentation score
 2888  * reaches within proactive compaction thresholds (as determined by the
 2889  * proactiveness tunable), it is possible that the function returns before
 2890  * reaching score targets due to various back-off conditions, such as,
 2891  * contention on per-node or per-zone locks.
 2892  */
 2893 static int compact_node(pg_data_t *pgdat, bool proactive)
 2894 {
 2895 	int zoneid;
 2896 	struct zone *zone;
 2897 	struct compact_control cc = {
 2898 		.order = -1,
 2899 		.mode = proactive ? MIGRATE_SYNC_LIGHT : MIGRATE_SYNC,
 2900 		.ignore_skip_hint = true,
 2901 		.whole_zone = true,
 2902 		.gfp_mask = GFP_KERNEL,
 2903 		.proactive_compaction = proactive,
 2904 	};
 2905 
 2906 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
 2907 		zone = &pgdat->node_zones[zoneid];
 2908 		if (!populated_zone(zone))
 2909 			continue;
 2910 
 2911 		if (fatal_signal_pending(current))
 2912 			return -EINTR;
 2913 
 2914 		cc.zone = zone;
 2915 
 2916 		compact_zone(&cc, NULL);
 2917 
 2918 		if (proactive) {
 2919 			count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
 2920 					     cc.total_migrate_scanned);
 2921 			count_compact_events(KCOMPACTD_FREE_SCANNED,
 2922 					     cc.total_free_scanned);
 2923 		}
 2924 	}
 2925 
 2926 	return 0;
 2927 }
 2928 
 2929 /* Compact all zones of all nodes in the system */
 2930 static int compact_nodes(void)
 2931 {
 2932 	int ret, nid;
 2933 
 2934 	/* Flush pending updates to the LRU lists */
 2935 	lru_add_drain_all();
 2936 
 2937 	for_each_online_node(nid) {
 2938 		ret = compact_node(NODE_DATA(nid), false);
 2939 		if (ret)
 2940 			return ret;
 2941 	}
 2942 
 2943 	return 0;
 2944 }
 2945 
 2946 static int compaction_proactiveness_sysctl_handler(const struct ctl_table *table, int write,
 2947 		void *buffer, size_t *length, loff_t *ppos)
 2948 {
 2949 	int rc, nid;
 2950 
 2951 	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
 2952 	if (rc)
 2953 		return rc;
 2954 
 2955 	if (write && sysctl_compaction_proactiveness) {
 2956 		for_each_online_node(nid) {
 2957 			pg_data_t *pgdat = NODE_DATA(nid);
 2958 
 2959 			if (pgdat->proactive_compact_trigger)
 2960 				continue;
 2961 
 2962 			pgdat->proactive_compact_trigger = true;
 2963 			trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1,
 2964 							     pgdat->nr_zones - 1);
 2965 			wake_up_interruptible(&pgdat->kcompactd_wait);
 2966 		}
 2967 	}
 2968 
 2969 	return 0;
 2970 }
 2971 
 2972 /*
 2973  * This is the entry point for compacting all nodes via
 2974  * /proc/sys/vm/compact_memory
 2975  */
 2976 static int sysctl_compaction_handler(const struct ctl_table *table, int write,
 2977 			void *buffer, size_t *length, loff_t *ppos)
 2978 {
 2979 	int ret;
 2980 
 2981 	ret = proc_dointvec(table, write, buffer, length, ppos);
 2982 	if (ret)
 2983 		return ret;
 2984 
 2985 	if (sysctl_compact_memory != 1)
 2986 		return -EINVAL;
 2987 
 2988 	if (write)
 2989 		ret = compact_nodes();
 2990 
 2991 	return ret;
 2992 }
 2993 
 2994 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
 2995 static ssize_t compact_store(struct device *dev,
 2996 			     struct device_attribute *attr,
 2997 			     const char *buf, size_t count)
 2998 {
 2999 	int nid = dev->id;
 3000 
 3001 	if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
 3002 		/* Flush pending updates to the LRU lists */
 3003 		lru_add_drain_all();
 3004 
 3005 		compact_node(NODE_DATA(nid), false);
 3006 	}
 3007 
 3008 	return count;
 3009 }
 3010 static DEVICE_ATTR_WO(compact);
 3011 
 3012 int compaction_register_node(struct node *node)
 3013 {
 3014 	return device_create_file(&node->dev, &dev_attr_compact);
 3015 }
 3016 
 3017 void compaction_unregister_node(struct node *node)
 3018 {
 3019 	device_remove_file(&node->dev, &dev_attr_compact);
 3020 }
 3021 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
 3022 
 3023 static inline bool kcompactd_work_requested(pg_data_t *pgdat)
 3024 {
 3025 	return pgdat->kcompactd_max_order > 0 || kthread_should_stop() ||
 3026 		pgdat->proactive_compact_trigger;
 3027 }
 3028 
 3029 static bool kcompactd_node_suitable(pg_data_t *pgdat)
 3030 {
 3031 	int zoneid;
 3032 	struct zone *zone;
 3033 	enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;
 3034 	enum compact_result ret;
 3035 	unsigned int alloc_flags = defrag_mode ?
 3036 		ALLOC_WMARK_HIGH : ALLOC_WMARK_MIN;
 3037 
 3038 	for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {
 3039 		zone = &pgdat->node_zones[zoneid];
 3040 
 3041 		if (!populated_zone(zone))
 3042 			continue;
 3043 
 3044 		ret = compaction_suit_allocation_order(zone,
 3045 				pgdat->kcompactd_max_order,
 3046 				highest_zoneidx, alloc_flags,
 3047 				false, true);
 3048 		if (ret == COMPACT_CONTINUE)
 3049 			return true;
 3050 	}
 3051 
 3052 	return false;
 3053 }
 3054 
 3055 static void kcompactd_do_work(pg_data_t *pgdat)
 3056 {
 3057 	/*
 3058 	 * With no special task, compact all zones so that a page of requested
 3059 	 * order is allocatable.
 3060 	 */
 3061 	int zoneid;
 3062 	struct zone *zone;
 3063 	struct compact_control cc = {
 3064 		.order = pgdat->kcompactd_max_order,
 3065 		.search_order = pgdat->kcompactd_max_order,
 3066 		.highest_zoneidx = pgdat->kcompactd_highest_zoneidx,
 3067 		.mode = MIGRATE_SYNC_LIGHT,
 3068 		.ignore_skip_hint = false,
 3069 		.gfp_mask = GFP_KERNEL,
 3070 		.alloc_flags = defrag_mode ? ALLOC_WMARK_HIGH : ALLOC_WMARK_MIN,
 3071 	};
 3072 	enum compact_result ret;
 3073 
 3074 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
 3075 							cc.highest_zoneidx);
 3076 	count_compact_event(KCOMPACTD_WAKE);
 3077 
 3078 	for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {
 3079 		int status;
 3080 
 3081 		zone = &pgdat->node_zones[zoneid];
 3082 		if (!populated_zone(zone))
 3083 			continue;
 3084 
 3085 		if (compaction_deferred(zone, cc.order))
 3086 			continue;
 3087 
 3088 		ret = compaction_suit_allocation_order(zone,
 3089 				cc.order, zoneid, cc.alloc_flags,
 3090 				false, true);
 3091 		if (ret != COMPACT_CONTINUE)
 3092 			continue;
 3093 
 3094 		if (kthread_should_stop())
 3095 			return;
 3096 
 3097 		cc.zone = zone;
 3098 		status = compact_zone(&cc, NULL);
 3099 
 3100 		if (status == COMPACT_SUCCESS) {
 3101 			compaction_defer_reset(zone, cc.order, false);
 3102 		} else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
 3103 			/*
 3104 			 * Buddy pages may become stranded on pcps that could
 3105 			 * otherwise coalesce on the zone's free area for
 3106 			 * order >= cc.order.  This is ratelimited by the
 3107 			 * upcoming deferral.
 3108 			 */
 3109 			drain_all_pages(zone);
 3110 
 3111 			/*
 3112 			 * We use sync migration mode here, so we defer like
 3113 			 * sync direct compaction does.
 3114 			 */
 3115 			defer_compaction(zone, cc.order);
 3116 		}
 3117 
 3118 		count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
 3119 				     cc.total_migrate_scanned);
 3120 		count_compact_events(KCOMPACTD_FREE_SCANNED,
 3121 				     cc.total_free_scanned);
 3122 	}
 3123 
 3124 	/*
 3125 	 * Regardless of success, we are done until woken up next. But remember
 3126 	 * the requested order/highest_zoneidx in case it was higher/tighter
 3127 	 * than our current ones
 3128 	 */
 3129 	if (pgdat->kcompactd_max_order <= cc.order)
 3130 		pgdat->kcompactd_max_order = 0;
 3131 	if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)
 3132 		pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
 3133 }
 3134 
 3135 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
 3136 {
 3137 	if (!order)
 3138 		return;
 3139 
 3140 	if (pgdat->kcompactd_max_order < order)
 3141 		pgdat->kcompactd_max_order = order;
 3142 
 3143 	if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)
 3144 		pgdat->kcompactd_highest_zoneidx = highest_zoneidx;
 3145 
 3146 	/*
 3147 	 * Pairs with implicit barrier in wait_event_freezable()
 3148 	 * such that wakeups are not missed.
 3149 	 */
 3150 	if (!wq_has_sleeper(&pgdat->kcompactd_wait))
 3151 		return;
 3152 
 3153 	if (!kcompactd_node_suitable(pgdat))
 3154 		return;
 3155 
 3156 	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
 3157 							highest_zoneidx);
 3158 	wake_up_interruptible(&pgdat->kcompactd_wait);
 3159 }
 3160 
 3161 /*
 3162  * The background compaction daemon, started as a kernel thread
 3163  * from the init process.
 3164  */
 3165 static int kcompactd(void *p)
 3166 {
 3167 	pg_data_t *pgdat = (pg_data_t *)p;
 3168 	long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);
 3169 	long timeout = default_timeout;
 3170 
 3171 	current->flags |= PF_KCOMPACTD;
 3172 	set_freezable();
 3173 
 3174 	pgdat->kcompactd_max_order = 0;
 3175 	pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
 3176 
 3177 	while (!kthread_should_stop()) {
 3178 		unsigned long pflags;
 3179 
 3180 		/*
 3181 		 * Avoid the unnecessary wakeup for proactive compaction
 3182 		 * when it is disabled.
 3183 		 */
 3184 		if (!sysctl_compaction_proactiveness)
 3185 			timeout = MAX_SCHEDULE_TIMEOUT;
 3186 		trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
 3187 		if (wait_event_freezable_timeout(pgdat->kcompactd_wait,
 3188 			kcompactd_work_requested(pgdat), timeout) &&
 3189 			!pgdat->proactive_compact_trigger) {
 3190 
 3191 			psi_memstall_enter(&pflags);
 3192 			kcompactd_do_work(pgdat);
 3193 			psi_memstall_leave(&pflags);
 3194 			/*
 3195 			 * Reset the timeout value. The defer timeout from
 3196 			 * proactive compaction is lost here but that is fine
 3197 			 * as the condition of the zone changing substantionally
 3198 			 * then carrying on with the previous defer interval is
 3199 			 * not useful.
 3200 			 */
 3201 			timeout = default_timeout;
 3202 			continue;
 3203 		}
 3204 
 3205 		/*
 3206 		 * Start the proactive work with default timeout. Based
 3207 		 * on the fragmentation score, this timeout is updated.
 3208 		 */
 3209 		timeout = default_timeout;
 3210 		if (should_proactive_compact_node(pgdat)) {
 3211 			unsigned int prev_score, score;
 3212 
 3213 			prev_score = fragmentation_score_node(pgdat);
 3214 			compact_node(pgdat, true);
 3215 			score = fragmentation_score_node(pgdat);
 3216 			/*
 3217 			 * Defer proactive compaction if the fragmentation
 3218 			 * score did not go down i.e. no progress made.
 3219 			 */
 3220 			if (unlikely(score >= prev_score))
 3221 				timeout =
 3222 				   default_timeout << COMPACT_MAX_DEFER_SHIFT;
 3223 		}
 3224 		if (unlikely(pgdat->proactive_compact_trigger))
 3225 			pgdat->proactive_compact_trigger = false;
 3226 	}
 3227 
 3228 	current->flags &= ~PF_KCOMPACTD;
 3229 
 3230 	return 0;
 3231 }
 3232 
 3233 /*
 3234  * This kcompactd start function will be called by init and node-hot-add.
 3235  * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
 3236  */
 3237 void __meminit kcompactd_run(int nid)
 3238 {
 3239 	pg_data_t *pgdat = NODE_DATA(nid);
 3240 
 3241 	if (pgdat->kcompactd)
 3242 		return;
 3243 
 3244 	pgdat->kcompactd = kthread_create_on_node(kcompactd, pgdat, nid, "kcompactd%d", nid);
 3245 	if (IS_ERR(pgdat->kcompactd)) {
 3246 		pr_err("Failed to start kcompactd on node %d\n", nid);
 3247 		pgdat->kcompactd = NULL;
 3248 	} else {
 3249 		wake_up_process(pgdat->kcompactd);
 3250 	}
 3251 }
 3252 
 3253 /*
 3254  * Called by memory hotplug when all memory in a node is offlined. Caller must
 3255  * be holding mem_hotplug_begin/done().
 3256  */
 3257 void __meminit kcompactd_stop(int nid)
 3258 {
 3259 	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
 3260 
 3261 	if (kcompactd) {
 3262 		kthread_stop(kcompactd);
 3263 		NODE_DATA(nid)->kcompactd = NULL;
 3264 	}
 3265 }
 3266 
 3267 static int proc_dointvec_minmax_warn_RT_change(const struct ctl_table *table,
 3268 		int write, void *buffer, size_t *lenp, loff_t *ppos)
 3269 {
 3270 	int ret, old;
 3271 
 3272 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)
 3273 		return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 3274 
 3275 	old = *(int *)table->data;
 3276 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 3277 	if (ret)
 3278 		return ret;
 3279 	if (old != *(int *)table->data)
 3280 		pr_warn_once("sysctl attribute %s changed by %s[%d]\n",
 3281 			     table->procname, current->comm,
 3282 			     task_pid_nr(current));
 3283 	return ret;
 3284 }
 3285 
 3286 static const struct ctl_table vm_compaction[] = {
 3287 	{
 3288 		.procname	= "compact_memory",
 3289 		.data		= &sysctl_compact_memory,
 3290 		.maxlen		= sizeof(int),
 3291 		.mode		= 0200,
 3292 		.proc_handler	= sysctl_compaction_handler,
 3293 	},
 3294 	{
 3295 		.procname	= "compaction_proactiveness",
 3296 		.data		= &sysctl_compaction_proactiveness,
 3297 		.maxlen		= sizeof(sysctl_compaction_proactiveness),
 3298 		.mode		= 0644,
 3299 		.proc_handler	= compaction_proactiveness_sysctl_handler,
 3300 		.extra1		= SYSCTL_ZERO,
 3301 		.extra2		= SYSCTL_ONE_HUNDRED,
 3302 	},
 3303 	{
 3304 		.procname	= "extfrag_threshold",
 3305 		.data		= &sysctl_extfrag_threshold,
 3306 		.maxlen		= sizeof(int),
 3307 		.mode		= 0644,
 3308 		.proc_handler	= proc_dointvec_minmax,
 3309 		.extra1		= SYSCTL_ZERO,
 3310 		.extra2		= SYSCTL_ONE_THOUSAND,
 3311 	},
 3312 	{
 3313 		.procname	= "compact_unevictable_allowed",
 3314 		.data		= &sysctl_compact_unevictable_allowed,
 3315 		.maxlen		= sizeof(int),
 3316 		.mode		= 0644,
 3317 		.proc_handler	= proc_dointvec_minmax_warn_RT_change,
 3318 		.extra1		= SYSCTL_ZERO,
 3319 		.extra2		= SYSCTL_ONE,
 3320 	},
 3321 };
 3322 
 3323 static int __init kcompactd_init(void)
 3324 {
 3325 	int nid;
 3326 
 3327 	for_each_node_state(nid, N_MEMORY)
 3328 		kcompactd_run(nid);
 3329 	register_sysctl_init("vm", vm_compaction);
 3330 	return 0;
 3331 }
 3332 subsys_initcall(kcompactd_init)
 3333 
 3334 #endif /* CONFIG_COMPACTION */