개념 설명 전체 · v6.18.37 / arch/arm64/mm/context.c

    1 // SPDX-License-Identifier: GPL-2.0-only
    2 /*
    3  * Based on arch/arm/mm/context.c
    4  *
    5  * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
    6  * Copyright (C) 2012 ARM Ltd.
    7  */
    8 
    9 #include <linux/bitfield.h>
   10 #include <linux/bitops.h>
   11 #include <linux/sched.h>
   12 #include <linux/slab.h>
   13 #include <linux/mm.h>
   14 
   15 #include <asm/cpufeature.h>
   16 #include <asm/mmu_context.h>
   17 #include <asm/smp.h>
   18 #include <asm/tlbflush.h>
   19 
   20 static u32 asid_bits;
   21 static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
   22 
   23 static atomic64_t asid_generation;
   24 static unsigned long *asid_map;
   25 
   26 static DEFINE_PER_CPU(atomic64_t, active_asids);
   27 static DEFINE_PER_CPU(u64, reserved_asids);
   28 static cpumask_t tlb_flush_pending;
   29 
   30 static unsigned long max_pinned_asids;
   31 static unsigned long nr_pinned_asids;
   32 static unsigned long *pinned_asid_map;
   33 
   34 #define ASID_MASK		(~GENMASK(asid_bits - 1, 0))
   35 #define ASID_FIRST_VERSION	(1UL << 16)
   36 
   37 #define NUM_USER_ASIDS		(1UL << asid_bits)
   38 #define ctxid2asid(asid)	((asid) & ~ASID_MASK)
   39 #define asid2ctxid(asid, genid)	((asid) | (genid))
   40 
   41 /* Get the ASIDBits supported by the current CPU */
   42 static u32 get_cpu_asid_bits(void)
   43 {
   44 	u32 asid;
   45 	int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1),
   46 						ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
   47 
   48 	switch (fld) {
   49 	default:
   50 		pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n",
   51 					smp_processor_id(),  fld);
   52 		fallthrough;
   53 	case ID_AA64MMFR0_EL1_ASIDBITS_8:
   54 		asid = 8;
   55 		break;
   56 	case ID_AA64MMFR0_EL1_ASIDBITS_16:
   57 		asid = 16;
   58 	}
   59 
   60 	return asid;
   61 }
   62 
   63 /* Check if the current cpu's ASIDBits is compatible with asid_bits */
   64 void verify_cpu_asid_bits(void)
   65 {
   66 	u32 asid = get_cpu_asid_bits();
   67 
   68 	if (asid < asid_bits) {
   69 		/*
   70 		 * We cannot decrease the ASID size at runtime, so panic if we support
   71 		 * fewer ASID bits than the boot CPU.
   72 		 */
   73 		pr_crit("CPU%d: smaller ASID size(%u) than boot CPU (%u)\n",
   74 				smp_processor_id(), asid, asid_bits);
   75 		cpu_panic_kernel();
   76 	}
   77 }
   78 
   79 static void set_kpti_asid_bits(unsigned long *map)
   80 {
   81 	unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);
   82 	/*
   83 	 * In case of KPTI kernel/user ASIDs are allocated in
   84 	 * pairs, the bottom bit distinguishes the two: if it
   85 	 * is set, then the ASID will map only userspace. Thus
   86 	 * mark even as reserved for kernel.
   87 	 */
   88 	memset(map, 0xaa, len);
   89 }
   90 
   91 static void set_reserved_asid_bits(void)
   92 {
   93 	if (pinned_asid_map)
   94 		bitmap_copy(asid_map, pinned_asid_map, NUM_USER_ASIDS);
   95 	else if (arm64_kernel_unmapped_at_el0())
   96 		set_kpti_asid_bits(asid_map);
   97 	else
   98 		bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
   99 }
  100 
  101 #define asid_gen_match(asid) \
  102 	(!(((asid) ^ atomic64_read(&asid_generation)) >> asid_bits))
  103 
  104 static void flush_context(void)
  105 {
  106 	int i;
  107 	u64 asid;
  108 
  109 	/* Update the list of reserved ASIDs and the ASID bitmap. */
  110 	set_reserved_asid_bits();
  111 
  112 	for_each_possible_cpu(i) {
  113 		asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
  114 		/*
  115 		 * If this CPU has already been through a
  116 		 * rollover, but hasn't run another task in
  117 		 * the meantime, we must preserve its reserved
  118 		 * ASID, as this is the only trace we have of
  119 		 * the process it is still running.
  120 		 */
  121 		if (asid == 0)
  122 			asid = per_cpu(reserved_asids, i);
  123 		__set_bit(ctxid2asid(asid), asid_map);
  124 		per_cpu(reserved_asids, i) = asid;
  125 	}
  126 
  127 	/*
  128 	 * Queue a TLB invalidation for each CPU to perform on next
  129 	 * context-switch
  130 	 */
  131 	cpumask_setall(&tlb_flush_pending);
  132 }
  133 
  134 static bool check_update_reserved_asid(u64 asid, u64 newasid)
  135 {
  136 	int cpu;
  137 	bool hit = false;
  138 
  139 	/*
  140 	 * Iterate over the set of reserved ASIDs looking for a match.
  141 	 * If we find one, then we can update our mm to use newasid
  142 	 * (i.e. the same ASID in the current generation) but we can't
  143 	 * exit the loop early, since we need to ensure that all copies
  144 	 * of the old ASID are updated to reflect the mm. Failure to do
  145 	 * so could result in us missing the reserved ASID in a future
  146 	 * generation.
  147 	 */
  148 	for_each_possible_cpu(cpu) {
  149 		if (per_cpu(reserved_asids, cpu) == asid) {
  150 			hit = true;
  151 			per_cpu(reserved_asids, cpu) = newasid;
  152 		}
  153 	}
  154 
  155 	return hit;
  156 }
  157 
  158 static u64 new_context(struct mm_struct *mm)
  159 {
  160 	static u32 cur_idx = 1;
  161 	u64 asid = atomic64_read(&mm->context.id);
  162 	u64 generation = atomic64_read(&asid_generation);
  163 
  164 	if (asid != 0) {
  165 		u64 newasid = asid2ctxid(ctxid2asid(asid), generation);
  166 
  167 		/*
  168 		 * If our current ASID was active during a rollover, we
  169 		 * can continue to use it and this was just a false alarm.
  170 		 */
  171 		if (check_update_reserved_asid(asid, newasid))
  172 			return newasid;
  173 
  174 		/*
  175 		 * If it is pinned, we can keep using it. Note that reserved
  176 		 * takes priority, because even if it is also pinned, we need to
  177 		 * update the generation into the reserved_asids.
  178 		 */
  179 		if (refcount_read(&mm->context.pinned))
  180 			return newasid;
  181 
  182 		/*
  183 		 * We had a valid ASID in a previous life, so try to re-use
  184 		 * it if possible.
  185 		 */
  186 		if (!__test_and_set_bit(ctxid2asid(asid), asid_map))
  187 			return newasid;
  188 	}
  189 
  190 	/*
  191 	 * Allocate a free ASID. If we can't find one, take a note of the
  192 	 * currently active ASIDs and mark the TLBs as requiring flushes.  We
  193 	 * always count from ASID #2 (index 1), as we use ASID #0 when setting
  194 	 * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
  195 	 * pairs.
  196 	 */
  197 	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
  198 	if (asid != NUM_USER_ASIDS)
  199 		goto set_asid;
  200 
  201 	/* We're out of ASIDs, so increment the global generation count */
  202 	generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION,
  203 						 &asid_generation);
  204 	flush_context();
  205 
  206 	/* We have more ASIDs than CPUs, so this will always succeed */
  207 	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
  208 
  209 set_asid:
  210 	__set_bit(asid, asid_map);
  211 	cur_idx = asid;
  212 	return asid2ctxid(asid, generation);
  213 }
  214 
  215 void check_and_switch_context(struct mm_struct *mm)
  216 {
  217 	unsigned long flags;
  218 	unsigned int cpu;
  219 	u64 asid, old_active_asid;
  220 
  221 	if (system_supports_cnp())
  222 		cpu_set_reserved_ttbr0();
  223 
  224 	asid = atomic64_read(&mm->context.id);
  225 
  226 	/*
  227 	 * The memory ordering here is subtle.
  228 	 * If our active_asids is non-zero and the ASID matches the current
  229 	 * generation, then we update the active_asids entry with a relaxed
  230 	 * cmpxchg. Racing with a concurrent rollover means that either:
  231 	 *
  232 	 * - We get a zero back from the cmpxchg and end up waiting on the
  233 	 *   lock. Taking the lock synchronises with the rollover and so
  234 	 *   we are forced to see the updated generation.
  235 	 *
  236 	 * - We get a valid ASID back from the cmpxchg, which means the
  237 	 *   relaxed xchg in flush_context will treat us as reserved
  238 	 *   because atomic RmWs are totally ordered for a given location.
  239 	 */
  240 	old_active_asid = atomic64_read(this_cpu_ptr(&active_asids));
  241 	if (old_active_asid && asid_gen_match(asid) &&
  242 	    atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_asids),
  243 				     old_active_asid, asid))
  244 		goto switch_mm_fastpath;
  245 
  246 	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  247 	/* Check that our ASID belongs to the current generation. */
  248 	asid = atomic64_read(&mm->context.id);
  249 	if (!asid_gen_match(asid)) {
  250 		asid = new_context(mm);
  251 		atomic64_set(&mm->context.id, asid);
  252 	}
  253 
  254 	cpu = smp_processor_id();
  255 	if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
  256 		local_flush_tlb_all();
  257 
  258 	atomic64_set(this_cpu_ptr(&active_asids), asid);
  259 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  260 
  261 switch_mm_fastpath:
  262 
  263 	arm64_apply_bp_hardening();
  264 
  265 	/*
  266 	 * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when
  267 	 * emulating PAN.
  268 	 */
  269 	if (!system_uses_ttbr0_pan())
  270 		cpu_switch_mm(mm->pgd, mm);
  271 }
  272 
  273 unsigned long arm64_mm_context_get(struct mm_struct *mm)
  274 {
  275 	unsigned long flags;
  276 	u64 asid;
  277 
  278 	if (!pinned_asid_map)
  279 		return 0;
  280 
  281 	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  282 
  283 	asid = atomic64_read(&mm->context.id);
  284 
  285 	if (refcount_inc_not_zero(&mm->context.pinned))
  286 		goto out_unlock;
  287 
  288 	if (nr_pinned_asids >= max_pinned_asids) {
  289 		asid = 0;
  290 		goto out_unlock;
  291 	}
  292 
  293 	if (!asid_gen_match(asid)) {
  294 		/*
  295 		 * We went through one or more rollover since that ASID was
  296 		 * used. Ensure that it is still valid, or generate a new one.
  297 		 */
  298 		asid = new_context(mm);
  299 		atomic64_set(&mm->context.id, asid);
  300 	}
  301 
  302 	nr_pinned_asids++;
  303 	__set_bit(ctxid2asid(asid), pinned_asid_map);
  304 	refcount_set(&mm->context.pinned, 1);
  305 
  306 out_unlock:
  307 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  308 
  309 	asid = ctxid2asid(asid);
  310 
  311 	/* Set the equivalent of USER_ASID_BIT */
  312 	if (asid && arm64_kernel_unmapped_at_el0())
  313 		asid |= 1;
  314 
  315 	return asid;
  316 }
  317 EXPORT_SYMBOL_GPL(arm64_mm_context_get);
  318 
  319 void arm64_mm_context_put(struct mm_struct *mm)
  320 {
  321 	unsigned long flags;
  322 	u64 asid = atomic64_read(&mm->context.id);
  323 
  324 	if (!pinned_asid_map)
  325 		return;
  326 
  327 	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  328 
  329 	if (refcount_dec_and_test(&mm->context.pinned)) {
  330 		__clear_bit(ctxid2asid(asid), pinned_asid_map);
  331 		nr_pinned_asids--;
  332 	}
  333 
  334 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  335 }
  336 EXPORT_SYMBOL_GPL(arm64_mm_context_put);
  337 
  338 /* Errata workaround post TTBRx_EL1 update. */
  339 asmlinkage void post_ttbr_update_workaround(void)
  340 {
  341 	if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456))
  342 		return;
  343 
  344 	asm(ALTERNATIVE("nop; nop; nop",
  345 			"ic iallu; dsb nsh; isb",
  346 			ARM64_WORKAROUND_CAVIUM_27456));
  347 }
  348 
  349 void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm)
  350 {
  351 	unsigned long ttbr1 = read_sysreg(ttbr1_el1);
  352 	unsigned long asid = ASID(mm);
  353 	unsigned long ttbr0 = phys_to_ttbr(pgd_phys);
  354 
  355 	/* Skip CNP for the reserved ASID */
  356 	if (system_supports_cnp() && asid)
  357 		ttbr0 |= TTBR_CNP_BIT;
  358 
  359 	/* SW PAN needs a copy of the ASID in TTBR0 for entry */
  360 	if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN))
  361 		ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid);
  362 
  363 	/* Set ASID in TTBR1 since TCR.A1 is set */
  364 	ttbr1 &= ~TTBR_ASID_MASK;
  365 	ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
  366 
  367 	cpu_set_reserved_ttbr0_nosync();
  368 	write_sysreg(ttbr1, ttbr1_el1);
  369 	write_sysreg(ttbr0, ttbr0_el1);
  370 	isb();
  371 	post_ttbr_update_workaround();
  372 }
  373 
  374 static int asids_update_limit(void)
  375 {
  376 	unsigned long num_available_asids = NUM_USER_ASIDS;
  377 
  378 	if (arm64_kernel_unmapped_at_el0()) {
  379 		num_available_asids /= 2;
  380 		if (pinned_asid_map)
  381 			set_kpti_asid_bits(pinned_asid_map);
  382 	}
  383 	/*
  384 	 * Expect allocation after rollover to fail if we don't have at least
  385 	 * one more ASID than CPUs. ASID #0 is reserved for init_mm.
  386 	 */
  387 	WARN_ON(num_available_asids - 1 <= num_possible_cpus());
  388 	pr_info("ASID allocator initialised with %lu entries\n",
  389 		num_available_asids);
  390 
  391 	/*
  392 	 * There must always be an ASID available after rollover. Ensure that,
  393 	 * even if all CPUs have a reserved ASID and the maximum number of ASIDs
  394 	 * are pinned, there still is at least one empty slot in the ASID map.
  395 	 */
  396 	max_pinned_asids = num_available_asids - num_possible_cpus() - 2;
  397 	return 0;
  398 }
  399 arch_initcall(asids_update_limit);
  400 
  401 static int asids_init(void)
  402 {
  403 	asid_bits = get_cpu_asid_bits();
  404 	atomic64_set(&asid_generation, ASID_FIRST_VERSION);
  405 	asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
  406 	if (!asid_map)
  407 		panic("Failed to allocate bitmap for %lu ASIDs\n",
  408 		      NUM_USER_ASIDS);
  409 
  410 	pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
  411 	nr_pinned_asids = 0;
  412 
  413 	/*
  414 	 * We cannot call set_reserved_asid_bits() here because CPU
  415 	 * caps are not finalized yet, so it is safer to assume KPTI
  416 	 * and reserve kernel ASID's from beginning.
  417 	 */
  418 	if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))
  419 		set_kpti_asid_bits(asid_map);
  420 	return 0;
  421 }
  422 early_initcall(asids_init);