개념 설명 전체 · v6.18.37 / drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

    1 // SPDX-License-Identifier: GPL-2.0
    2 /*
    3  * IOMMU API for ARM architected SMMUv3 implementations.
    4  *
    5  * Copyright (C) 2015 ARM Limited
    6  *
    7  * Author: Will Deacon <[email protected]>
    8  *
    9  * This driver is powered by bad coffee and bombay mix.
   10  */
   11 
   12 #include <linux/acpi.h>
   13 #include <linux/acpi_iort.h>
   14 #include <linux/bitops.h>
   15 #include <linux/crash_dump.h>
   16 #include <linux/delay.h>
   17 #include <linux/err.h>
   18 #include <linux/interrupt.h>
   19 #include <linux/io-pgtable.h>
   20 #include <linux/iopoll.h>
   21 #include <linux/module.h>
   22 #include <linux/msi.h>
   23 #include <linux/of.h>
   24 #include <linux/of_address.h>
   25 #include <linux/of_platform.h>
   26 #include <linux/pci.h>
   27 #include <linux/pci-ats.h>
   28 #include <linux/platform_device.h>
   29 #include <linux/string_choices.h>
   30 #include <kunit/visibility.h>
   31 #include <uapi/linux/iommufd.h>
   32 
   33 #include "arm-smmu-v3.h"
   34 #include "../../dma-iommu.h"
   35 
   36 static bool disable_msipolling;
   37 module_param(disable_msipolling, bool, 0444);
   38 MODULE_PARM_DESC(disable_msipolling,
   39 	"Disable MSI-based polling for CMD_SYNC completion.");
   40 
   41 static const struct iommu_ops arm_smmu_ops;
   42 static struct iommu_dirty_ops arm_smmu_dirty_ops;
   43 
   44 enum arm_smmu_msi_index {
   45 	EVTQ_MSI_INDEX,
   46 	GERROR_MSI_INDEX,
   47 	PRIQ_MSI_INDEX,
   48 	ARM_SMMU_MAX_MSIS,
   49 };
   50 
   51 #define NUM_ENTRY_QWORDS 8
   52 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
   53 static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
   54 
   55 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
   56 	[EVTQ_MSI_INDEX] = {
   57 		ARM_SMMU_EVTQ_IRQ_CFG0,
   58 		ARM_SMMU_EVTQ_IRQ_CFG1,
   59 		ARM_SMMU_EVTQ_IRQ_CFG2,
   60 	},
   61 	[GERROR_MSI_INDEX] = {
   62 		ARM_SMMU_GERROR_IRQ_CFG0,
   63 		ARM_SMMU_GERROR_IRQ_CFG1,
   64 		ARM_SMMU_GERROR_IRQ_CFG2,
   65 	},
   66 	[PRIQ_MSI_INDEX] = {
   67 		ARM_SMMU_PRIQ_IRQ_CFG0,
   68 		ARM_SMMU_PRIQ_IRQ_CFG1,
   69 		ARM_SMMU_PRIQ_IRQ_CFG2,
   70 	},
   71 };
   72 
   73 struct arm_smmu_option_prop {
   74 	u32 opt;
   75 	const char *prop;
   76 };
   77 
   78 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
   79 DEFINE_MUTEX(arm_smmu_asid_lock);
   80 
   81 static struct arm_smmu_option_prop arm_smmu_options[] = {
   82 	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
   83 	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
   84 	{ 0, NULL},
   85 };
   86 
   87 static const char * const event_str[] = {
   88 	[EVT_ID_BAD_STREAMID_CONFIG] = "C_BAD_STREAMID",
   89 	[EVT_ID_STE_FETCH_FAULT] = "F_STE_FETCH",
   90 	[EVT_ID_BAD_STE_CONFIG] = "C_BAD_STE",
   91 	[EVT_ID_STREAM_DISABLED_FAULT] = "F_STREAM_DISABLED",
   92 	[EVT_ID_BAD_SUBSTREAMID_CONFIG] = "C_BAD_SUBSTREAMID",
   93 	[EVT_ID_CD_FETCH_FAULT] = "F_CD_FETCH",
   94 	[EVT_ID_BAD_CD_CONFIG] = "C_BAD_CD",
   95 	[EVT_ID_TRANSLATION_FAULT] = "F_TRANSLATION",
   96 	[EVT_ID_ADDR_SIZE_FAULT] = "F_ADDR_SIZE",
   97 	[EVT_ID_ACCESS_FAULT] = "F_ACCESS",
   98 	[EVT_ID_PERMISSION_FAULT] = "F_PERMISSION",
   99 	[EVT_ID_VMS_FETCH_FAULT] = "F_VMS_FETCH",
  100 };
  101 
  102 static const char * const event_class_str[] = {
  103 	[0] = "CD fetch",
  104 	[1] = "Stage 1 translation table fetch",
  105 	[2] = "Input address caused fault",
  106 	[3] = "Reserved",
  107 };
  108 
  109 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
  110 
  111 static void parse_driver_options(struct arm_smmu_device *smmu)
  112 {
  113 	int i = 0;
  114 
  115 	do {
  116 		if (of_property_read_bool(smmu->dev->of_node,
  117 						arm_smmu_options[i].prop)) {
  118 			smmu->options |= arm_smmu_options[i].opt;
  119 			dev_notice(smmu->dev, "option %s\n",
  120 				arm_smmu_options[i].prop);
  121 		}
  122 	} while (arm_smmu_options[++i].opt);
  123 }
  124 
  125 /* Low-level queue manipulation functions */
  126 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
  127 {
  128 	u32 space, prod, cons;
  129 
  130 	prod = Q_IDX(q, q->prod);
  131 	cons = Q_IDX(q, q->cons);
  132 
  133 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
  134 		space = (1 << q->max_n_shift) - (prod - cons);
  135 	else
  136 		space = cons - prod;
  137 
  138 	return space >= n;
  139 }
  140 
  141 static bool queue_full(struct arm_smmu_ll_queue *q)
  142 {
  143 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
  144 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
  145 }
  146 
  147 static bool queue_empty(struct arm_smmu_ll_queue *q)
  148 {
  149 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
  150 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
  151 }
  152 
  153 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
  154 {
  155 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
  156 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
  157 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
  158 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
  159 }
  160 
  161 static void queue_sync_cons_out(struct arm_smmu_queue *q)
  162 {
  163 	/*
  164 	 * Ensure that all CPU accesses (reads and writes) to the queue
  165 	 * are complete before we update the cons pointer.
  166 	 */
  167 	__iomb();
  168 	writel_relaxed(q->llq.cons, q->cons_reg);
  169 }
  170 
  171 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
  172 {
  173 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
  174 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
  175 }
  176 
  177 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
  178 {
  179 	struct arm_smmu_ll_queue *llq = &q->llq;
  180 
  181 	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
  182 		return;
  183 
  184 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
  185 		      Q_IDX(llq, llq->cons);
  186 	queue_sync_cons_out(q);
  187 }
  188 
  189 static int queue_sync_prod_in(struct arm_smmu_queue *q)
  190 {
  191 	u32 prod;
  192 	int ret = 0;
  193 
  194 	/*
  195 	 * We can't use the _relaxed() variant here, as we must prevent
  196 	 * speculative reads of the queue before we have determined that
  197 	 * prod has indeed moved.
  198 	 */
  199 	prod = readl(q->prod_reg);
  200 
  201 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))
  202 		ret = -EOVERFLOW;
  203 
  204 	q->llq.prod = prod;
  205 	return ret;
  206 }
  207 
  208 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
  209 {
  210 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
  211 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
  212 }
  213 
  214 static void queue_poll_init(struct arm_smmu_device *smmu,
  215 			    struct arm_smmu_queue_poll *qp)
  216 {
  217 	qp->delay = 1;
  218 	qp->spin_cnt = 0;
  219 	qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
  220 	qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
  221 }
  222 
  223 static int queue_poll(struct arm_smmu_queue_poll *qp)
  224 {
  225 	if (ktime_compare(ktime_get(), qp->timeout) > 0)
  226 		return -ETIMEDOUT;
  227 
  228 	if (qp->wfe) {
  229 		wfe();
  230 	} else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
  231 		cpu_relax();
  232 	} else {
  233 		udelay(qp->delay);
  234 		qp->delay *= 2;
  235 		qp->spin_cnt = 0;
  236 	}
  237 
  238 	return 0;
  239 }
  240 
  241 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
  242 {
  243 	int i;
  244 
  245 	for (i = 0; i < n_dwords; ++i)
  246 		*dst++ = cpu_to_le64(*src++);
  247 }
  248 
  249 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
  250 {
  251 	int i;
  252 
  253 	for (i = 0; i < n_dwords; ++i)
  254 		*dst++ = le64_to_cpu(*src++);
  255 }
  256 
  257 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
  258 {
  259 	if (queue_empty(&q->llq))
  260 		return -EAGAIN;
  261 
  262 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
  263 	queue_inc_cons(&q->llq);
  264 	queue_sync_cons_out(q);
  265 	return 0;
  266 }
  267 
  268 /* High-level queue accessors */
  269 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
  270 {
  271 	memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
  272 	cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
  273 
  274 	switch (ent->opcode) {
  275 	case CMDQ_OP_TLBI_EL2_ALL:
  276 	case CMDQ_OP_TLBI_NSNH_ALL:
  277 		break;
  278 	case CMDQ_OP_PREFETCH_CFG:
  279 		cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
  280 		break;
  281 	case CMDQ_OP_CFGI_CD:
  282 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
  283 		fallthrough;
  284 	case CMDQ_OP_CFGI_STE:
  285 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
  286 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
  287 		break;
  288 	case CMDQ_OP_CFGI_CD_ALL:
  289 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
  290 		break;
  291 	case CMDQ_OP_CFGI_ALL:
  292 		/* Cover the entire SID range */
  293 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
  294 		break;
  295 	case CMDQ_OP_TLBI_NH_VA:
  296 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
  297 		fallthrough;
  298 	case CMDQ_OP_TLBI_EL2_VA:
  299 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
  300 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
  301 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
  302 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
  303 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
  304 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
  305 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
  306 		break;
  307 	case CMDQ_OP_TLBI_S2_IPA:
  308 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
  309 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
  310 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
  311 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
  312 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
  313 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
  314 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
  315 		break;
  316 	case CMDQ_OP_TLBI_NH_ASID:
  317 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
  318 		fallthrough;
  319 	case CMDQ_OP_TLBI_NH_ALL:
  320 	case CMDQ_OP_TLBI_S12_VMALL:
  321 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
  322 		break;
  323 	case CMDQ_OP_TLBI_EL2_ASID:
  324 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
  325 		break;
  326 	case CMDQ_OP_ATC_INV:
  327 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
  328 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
  329 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
  330 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
  331 		cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
  332 		cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
  333 		break;
  334 	case CMDQ_OP_PRI_RESP:
  335 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
  336 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
  337 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
  338 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
  339 		switch (ent->pri.resp) {
  340 		case PRI_RESP_DENY:
  341 		case PRI_RESP_FAIL:
  342 		case PRI_RESP_SUCC:
  343 			break;
  344 		default:
  345 			return -EINVAL;
  346 		}
  347 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
  348 		break;
  349 	case CMDQ_OP_RESUME:
  350 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
  351 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
  352 		cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
  353 		break;
  354 	case CMDQ_OP_CMD_SYNC:
  355 		if (ent->sync.msiaddr) {
  356 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
  357 			cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
  358 		} else {
  359 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
  360 		}
  361 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
  362 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
  363 		break;
  364 	default:
  365 		return -ENOENT;
  366 	}
  367 
  368 	return 0;
  369 }
  370 
  371 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
  372 					       struct arm_smmu_cmdq_ent *ent)
  373 {
  374 	struct arm_smmu_cmdq *cmdq = NULL;
  375 
  376 	if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
  377 		cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
  378 
  379 	return cmdq ?: &smmu->cmdq;
  380 }
  381 
  382 static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
  383 					     struct arm_smmu_cmdq *cmdq)
  384 {
  385 	if (cmdq == &smmu->cmdq)
  386 		return false;
  387 
  388 	return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
  389 }
  390 
  391 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
  392 					 struct arm_smmu_cmdq *cmdq, u32 prod)
  393 {
  394 	struct arm_smmu_queue *q = &cmdq->q;
  395 	struct arm_smmu_cmdq_ent ent = {
  396 		.opcode = CMDQ_OP_CMD_SYNC,
  397 	};
  398 
  399 	/*
  400 	 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
  401 	 * payload, so the write will zero the entire command on that platform.
  402 	 */
  403 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
  404 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
  405 				   q->ent_dwords * 8;
  406 	}
  407 
  408 	arm_smmu_cmdq_build_cmd(cmd, &ent);
  409 	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
  410 		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
  411 }
  412 
  413 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
  414 			      struct arm_smmu_cmdq *cmdq)
  415 {
  416 	static const char * const cerror_str[] = {
  417 		[CMDQ_ERR_CERROR_NONE_IDX]	= "No error",
  418 		[CMDQ_ERR_CERROR_ILL_IDX]	= "Illegal command",
  419 		[CMDQ_ERR_CERROR_ABT_IDX]	= "Abort on command fetch",
  420 		[CMDQ_ERR_CERROR_ATC_INV_IDX]	= "ATC invalidate timeout",
  421 	};
  422 	struct arm_smmu_queue *q = &cmdq->q;
  423 
  424 	int i;
  425 	u64 cmd[CMDQ_ENT_DWORDS];
  426 	u32 cons = readl_relaxed(q->cons_reg);
  427 	u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
  428 	struct arm_smmu_cmdq_ent cmd_sync = {
  429 		.opcode = CMDQ_OP_CMD_SYNC,
  430 	};
  431 
  432 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
  433 		idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
  434 
  435 	switch (idx) {
  436 	case CMDQ_ERR_CERROR_ABT_IDX:
  437 		dev_err(smmu->dev, "retrying command fetch\n");
  438 		return;
  439 	case CMDQ_ERR_CERROR_NONE_IDX:
  440 		return;
  441 	case CMDQ_ERR_CERROR_ATC_INV_IDX:
  442 		/*
  443 		 * ATC Invalidation Completion timeout. CONS is still pointing
  444 		 * at the CMD_SYNC. Attempt to complete other pending commands
  445 		 * by repeating the CMD_SYNC, though we might well end up back
  446 		 * here since the ATC invalidation may still be pending.
  447 		 */
  448 		return;
  449 	case CMDQ_ERR_CERROR_ILL_IDX:
  450 	default:
  451 		break;
  452 	}
  453 
  454 	/*
  455 	 * We may have concurrent producers, so we need to be careful
  456 	 * not to touch any of the shadow cmdq state.
  457 	 */
  458 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
  459 	dev_err(smmu->dev, "skipping command in error state:\n");
  460 	for (i = 0; i < ARRAY_SIZE(cmd); ++i)
  461 		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
  462 
  463 	/* Convert the erroneous command into a CMD_SYNC */
  464 	arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
  465 	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
  466 		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
  467 
  468 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
  469 }
  470 
  471 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
  472 {
  473 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
  474 }
  475 
  476 /*
  477  * Command queue locking.
  478  * This is a form of bastardised rwlock with the following major changes:
  479  *
  480  * - The only LOCK routines are exclusive_trylock() and shared_lock().
  481  *   Neither have barrier semantics, and instead provide only a control
  482  *   dependency.
  483  *
  484  * - The UNLOCK routines are supplemented with shared_tryunlock(), which
  485  *   fails if the caller appears to be the last lock holder (yes, this is
  486  *   racy). All successful UNLOCK routines have RELEASE semantics.
  487  */
  488 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
  489 {
  490 	/*
  491 	 * When held in exclusive state, the lock counter is set to INT_MIN
  492 	 * so these increments won't hurt as the value will remain negative.
  493 	 * The increment will also signal the exclusive locker that there are
  494 	 * shared waiters.
  495 	 */
  496 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
  497 		return;
  498 
  499 	/*
  500 	 * Someone else is holding the lock in exclusive state, so wait
  501 	 * for them to finish. Since we already incremented the lock counter,
  502 	 * no exclusive lock can be acquired until we finish. We don't need
  503 	 * the return value since we only care that the exclusive lock is
  504 	 * released (i.e. the lock counter is non-negative).
  505 	 * Once the exclusive locker releases the lock, the sign bit will
  506 	 * be cleared and our increment will make the lock counter positive,
  507 	 * allowing us to proceed.
  508 	 */
  509 	atomic_cond_read_relaxed(&cmdq->lock, VAL > 0);
  510 }
  511 
  512 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
  513 {
  514 	(void)atomic_dec_return_release(&cmdq->lock);
  515 }
  516 
  517 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
  518 {
  519 	if (atomic_read(&cmdq->lock) == 1)
  520 		return false;
  521 
  522 	arm_smmu_cmdq_shared_unlock(cmdq);
  523 	return true;
  524 }
  525 
  526 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)		\
  527 ({									\
  528 	bool __ret;							\
  529 	local_irq_save(flags);						\
  530 	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
  531 	if (!__ret)							\
  532 		local_irq_restore(flags);				\
  533 	__ret;								\
  534 })
  535 
  536 /*
  537  * Only clear the sign bit when releasing the exclusive lock this will
  538  * allow any shared_lock() waiters to proceed without the possibility
  539  * of entering the exclusive lock in a tight loop.
  540  */
  541 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\
  542 ({									\
  543 	atomic_fetch_andnot_release(INT_MIN, &cmdq->lock);		\
  544 	local_irq_restore(flags);					\
  545 })
  546 
  547 
  548 /*
  549  * Command queue insertion.
  550  * This is made fiddly by our attempts to achieve some sort of scalability
  551  * since there is one queue shared amongst all of the CPUs in the system.  If
  552  * you like mixed-size concurrency, dependency ordering and relaxed atomics,
  553  * then you'll *love* this monstrosity.
  554  *
  555  * The basic idea is to split the queue up into ranges of commands that are
  556  * owned by a given CPU; the owner may not have written all of the commands
  557  * itself, but is responsible for advancing the hardware prod pointer when
  558  * the time comes. The algorithm is roughly:
  559  *
  560  * 	1. Allocate some space in the queue. At this point we also discover
  561  *	   whether the head of the queue is currently owned by another CPU,
  562  *	   or whether we are the owner.
  563  *
  564  *	2. Write our commands into our allocated slots in the queue.
  565  *
  566  *	3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
  567  *
  568  *	4. If we are an owner:
  569  *		a. Wait for the previous owner to finish.
  570  *		b. Mark the queue head as unowned, which tells us the range
  571  *		   that we are responsible for publishing.
  572  *		c. Wait for all commands in our owned range to become valid.
  573  *		d. Advance the hardware prod pointer.
  574  *		e. Tell the next owner we've finished.
  575  *
  576  *	5. If we are inserting a CMD_SYNC (we may or may not have been an
  577  *	   owner), then we need to stick around until it has completed:
  578  *		a. If we have MSIs, the SMMU can write back into the CMD_SYNC
  579  *		   to clear the first 4 bytes.
  580  *		b. Otherwise, we spin waiting for the hardware cons pointer to
  581  *		   advance past our command.
  582  *
  583  * The devil is in the details, particularly the use of locking for handling
  584  * SYNC completion and freeing up space in the queue before we think that it is
  585  * full.
  586  */
  587 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
  588 					       u32 sprod, u32 eprod, bool set)
  589 {
  590 	u32 swidx, sbidx, ewidx, ebidx;
  591 	struct arm_smmu_ll_queue llq = {
  592 		.max_n_shift	= cmdq->q.llq.max_n_shift,
  593 		.prod		= sprod,
  594 	};
  595 
  596 	ewidx = BIT_WORD(Q_IDX(&llq, eprod));
  597 	ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
  598 
  599 	while (llq.prod != eprod) {
  600 		unsigned long mask;
  601 		atomic_long_t *ptr;
  602 		u32 limit = BITS_PER_LONG;
  603 
  604 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
  605 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
  606 
  607 		ptr = &cmdq->valid_map[swidx];
  608 
  609 		if ((swidx == ewidx) && (sbidx < ebidx))
  610 			limit = ebidx;
  611 
  612 		mask = GENMASK(limit - 1, sbidx);
  613 
  614 		/*
  615 		 * The valid bit is the inverse of the wrap bit. This means
  616 		 * that a zero-initialised queue is invalid and, after marking
  617 		 * all entries as valid, they become invalid again when we
  618 		 * wrap.
  619 		 */
  620 		if (set) {
  621 			atomic_long_xor(mask, ptr);
  622 		} else { /* Poll */
  623 			unsigned long valid;
  624 
  625 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
  626 			atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
  627 		}
  628 
  629 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
  630 	}
  631 }
  632 
  633 /* Mark all entries in the range [sprod, eprod) as valid */
  634 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
  635 					u32 sprod, u32 eprod)
  636 {
  637 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
  638 }
  639 
  640 /* Wait for all entries in the range [sprod, eprod) to become valid */
  641 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
  642 					 u32 sprod, u32 eprod)
  643 {
  644 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
  645 }
  646 
  647 /* Wait for the command queue to become non-full */
  648 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
  649 					     struct arm_smmu_cmdq *cmdq,
  650 					     struct arm_smmu_ll_queue *llq)
  651 {
  652 	unsigned long flags;
  653 	struct arm_smmu_queue_poll qp;
  654 	int ret = 0;
  655 
  656 	/*
  657 	 * Try to update our copy of cons by grabbing exclusive cmdq access. If
  658 	 * that fails, spin until somebody else updates it for us.
  659 	 */
  660 	if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
  661 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
  662 		arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
  663 		llq->val = READ_ONCE(cmdq->q.llq.val);
  664 		return 0;
  665 	}
  666 
  667 	queue_poll_init(smmu, &qp);
  668 	do {
  669 		llq->val = READ_ONCE(cmdq->q.llq.val);
  670 		if (!queue_full(llq))
  671 			break;
  672 
  673 		ret = queue_poll(&qp);
  674 	} while (!ret);
  675 
  676 	return ret;
  677 }
  678 
  679 /*
  680  * Wait until the SMMU signals a CMD_SYNC completion MSI.
  681  * Must be called with the cmdq lock held in some capacity.
  682  */
  683 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
  684 					  struct arm_smmu_cmdq *cmdq,
  685 					  struct arm_smmu_ll_queue *llq)
  686 {
  687 	int ret = 0;
  688 	struct arm_smmu_queue_poll qp;
  689 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
  690 
  691 	queue_poll_init(smmu, &qp);
  692 
  693 	/*
  694 	 * The MSI won't generate an event, since it's being written back
  695 	 * into the command queue.
  696 	 */
  697 	qp.wfe = false;
  698 	smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
  699 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
  700 	return ret;
  701 }
  702 
  703 /*
  704  * Wait until the SMMU cons index passes llq->prod.
  705  * Must be called with the cmdq lock held in some capacity.
  706  */
  707 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
  708 					       struct arm_smmu_cmdq *cmdq,
  709 					       struct arm_smmu_ll_queue *llq)
  710 {
  711 	struct arm_smmu_queue_poll qp;
  712 	u32 prod = llq->prod;
  713 	int ret = 0;
  714 
  715 	queue_poll_init(smmu, &qp);
  716 	llq->val = READ_ONCE(cmdq->q.llq.val);
  717 	do {
  718 		if (queue_consumed(llq, prod))
  719 			break;
  720 
  721 		ret = queue_poll(&qp);
  722 
  723 		/*
  724 		 * This needs to be a readl() so that our subsequent call
  725 		 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
  726 		 *
  727 		 * Specifically, we need to ensure that we observe all
  728 		 * shared_lock()s by other CMD_SYNCs that share our owner,
  729 		 * so that a failing call to tryunlock() means that we're
  730 		 * the last one out and therefore we can safely advance
  731 		 * cmdq->q.llq.cons. Roughly speaking:
  732 		 *
  733 		 * CPU 0		CPU1			CPU2 (us)
  734 		 *
  735 		 * if (sync)
  736 		 * 	shared_lock();
  737 		 *
  738 		 * dma_wmb();
  739 		 * set_valid_map();
  740 		 *
  741 		 * 			if (owner) {
  742 		 *				poll_valid_map();
  743 		 *				<control dependency>
  744 		 *				writel(prod_reg);
  745 		 *
  746 		 *						readl(cons_reg);
  747 		 *						tryunlock();
  748 		 *
  749 		 * Requires us to see CPU 0's shared_lock() acquisition.
  750 		 */
  751 		llq->cons = readl(cmdq->q.cons_reg);
  752 	} while (!ret);
  753 
  754 	return ret;
  755 }
  756 
  757 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
  758 					 struct arm_smmu_cmdq *cmdq,
  759 					 struct arm_smmu_ll_queue *llq)
  760 {
  761 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
  762 	    !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
  763 		return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
  764 
  765 	return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
  766 }
  767 
  768 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
  769 					u32 prod, int n)
  770 {
  771 	int i;
  772 	struct arm_smmu_ll_queue llq = {
  773 		.max_n_shift	= cmdq->q.llq.max_n_shift,
  774 		.prod		= prod,
  775 	};
  776 
  777 	for (i = 0; i < n; ++i) {
  778 		u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
  779 
  780 		prod = queue_inc_prod_n(&llq, i);
  781 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
  782 	}
  783 }
  784 
  785 /*
  786  * This is the actual insertion function, and provides the following
  787  * ordering guarantees to callers:
  788  *
  789  * - There is a dma_wmb() before publishing any commands to the queue.
  790  *   This can be relied upon to order prior writes to data structures
  791  *   in memory (such as a CD or an STE) before the command.
  792  *
  793  * - On completion of a CMD_SYNC, there is a control dependency.
  794  *   This can be relied upon to order subsequent writes to memory (e.g.
  795  *   freeing an IOVA) after completion of the CMD_SYNC.
  796  *
  797  * - Command insertion is totally ordered, so if two CPUs each race to
  798  *   insert their own list of commands then all of the commands from one
  799  *   CPU will appear before any of the commands from the other CPU.
  800  */
  801 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
  802 				struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
  803 				bool sync)
  804 {
  805 	u64 cmd_sync[CMDQ_ENT_DWORDS];
  806 	u32 prod;
  807 	unsigned long flags;
  808 	bool owner;
  809 	struct arm_smmu_ll_queue llq, head;
  810 	int ret = 0;
  811 
  812 	llq.max_n_shift = cmdq->q.llq.max_n_shift;
  813 
  814 	/* 1. Allocate some space in the queue */
  815 	local_irq_save(flags);
  816 	llq.val = READ_ONCE(cmdq->q.llq.val);
  817 	do {
  818 		u64 old;
  819 
  820 		while (!queue_has_space(&llq, n + sync)) {
  821 			local_irq_restore(flags);
  822 			if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
  823 				dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
  824 			local_irq_save(flags);
  825 		}
  826 
  827 		head.cons = llq.cons;
  828 		head.prod = queue_inc_prod_n(&llq, n + sync) |
  829 					     CMDQ_PROD_OWNED_FLAG;
  830 
  831 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
  832 		if (old == llq.val)
  833 			break;
  834 
  835 		llq.val = old;
  836 	} while (1);
  837 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
  838 	head.prod &= ~CMDQ_PROD_OWNED_FLAG;
  839 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
  840 
  841 	/*
  842 	 * 2. Write our commands into the queue
  843 	 * Dependency ordering from the cmpxchg() loop above.
  844 	 */
  845 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
  846 	if (sync) {
  847 		prod = queue_inc_prod_n(&llq, n);
  848 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
  849 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
  850 
  851 		/*
  852 		 * In order to determine completion of our CMD_SYNC, we must
  853 		 * ensure that the queue can't wrap twice without us noticing.
  854 		 * We achieve that by taking the cmdq lock as shared before
  855 		 * marking our slot as valid.
  856 		 */
  857 		arm_smmu_cmdq_shared_lock(cmdq);
  858 	}
  859 
  860 	/* 3. Mark our slots as valid, ensuring commands are visible first */
  861 	dma_wmb();
  862 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
  863 
  864 	/* 4. If we are the owner, take control of the SMMU hardware */
  865 	if (owner) {
  866 		/* a. Wait for previous owner to finish */
  867 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
  868 
  869 		/* b. Stop gathering work by clearing the owned flag */
  870 		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
  871 						   &cmdq->q.llq.atomic.prod);
  872 		prod &= ~CMDQ_PROD_OWNED_FLAG;
  873 
  874 		/*
  875 		 * c. Wait for any gathered work to be written to the queue.
  876 		 * Note that we read our own entries so that we have the control
  877 		 * dependency required by (d).
  878 		 */
  879 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
  880 
  881 		/*
  882 		 * d. Advance the hardware prod pointer
  883 		 * Control dependency ordering from the entries becoming valid.
  884 		 */
  885 		writel_relaxed(prod, cmdq->q.prod_reg);
  886 
  887 		/*
  888 		 * e. Tell the next owner we're done
  889 		 * Make sure we've updated the hardware first, so that we don't
  890 		 * race to update prod and potentially move it backwards.
  891 		 */
  892 		atomic_set_release(&cmdq->owner_prod, prod);
  893 	}
  894 
  895 	/* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
  896 	if (sync) {
  897 		llq.prod = queue_inc_prod_n(&llq, n);
  898 		ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
  899 		if (ret) {
  900 			dev_err_ratelimited(smmu->dev,
  901 					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
  902 					    llq.prod,
  903 					    readl_relaxed(cmdq->q.prod_reg),
  904 					    readl_relaxed(cmdq->q.cons_reg));
  905 		}
  906 
  907 		/*
  908 		 * Try to unlock the cmdq lock. This will fail if we're the last
  909 		 * reader, in which case we can safely update cmdq->q.llq.cons
  910 		 */
  911 		if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
  912 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
  913 			arm_smmu_cmdq_shared_unlock(cmdq);
  914 		}
  915 	}
  916 
  917 	local_irq_restore(flags);
  918 	return ret;
  919 }
  920 
  921 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
  922 				     struct arm_smmu_cmdq_ent *ent,
  923 				     bool sync)
  924 {
  925 	u64 cmd[CMDQ_ENT_DWORDS];
  926 
  927 	if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
  928 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
  929 			 ent->opcode);
  930 		return -EINVAL;
  931 	}
  932 
  933 	return arm_smmu_cmdq_issue_cmdlist(
  934 		smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
  935 }
  936 
  937 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
  938 				   struct arm_smmu_cmdq_ent *ent)
  939 {
  940 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
  941 }
  942 
  943 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
  944 					     struct arm_smmu_cmdq_ent *ent)
  945 {
  946 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
  947 }
  948 
  949 static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
  950 				     struct arm_smmu_cmdq_batch *cmds,
  951 				     struct arm_smmu_cmdq_ent *ent)
  952 {
  953 	cmds->num = 0;
  954 	cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
  955 }
  956 
  957 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
  958 				    struct arm_smmu_cmdq_batch *cmds,
  959 				    struct arm_smmu_cmdq_ent *cmd)
  960 {
  961 	bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
  962 	bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
  963 			  (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
  964 	int index;
  965 
  966 	if (force_sync || unsupported_cmd) {
  967 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
  968 					    cmds->num, true);
  969 		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
  970 	}
  971 
  972 	if (cmds->num == CMDQ_BATCH_ENTRIES) {
  973 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
  974 					    cmds->num, false);
  975 		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
  976 	}
  977 
  978 	index = cmds->num * CMDQ_ENT_DWORDS;
  979 	if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
  980 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
  981 			 cmd->opcode);
  982 		return;
  983 	}
  984 
  985 	cmds->num++;
  986 }
  987 
  988 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
  989 				      struct arm_smmu_cmdq_batch *cmds)
  990 {
  991 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
  992 					   cmds->num, true);
  993 }
  994 
  995 static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
  996 				   struct iommu_page_response *resp)
  997 {
  998 	struct arm_smmu_cmdq_ent cmd = {0};
  999 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 1000 	int sid = master->streams[0].id;
 1001 
 1002 	if (WARN_ON(!master->stall_enabled))
 1003 		return;
 1004 
 1005 	cmd.opcode		= CMDQ_OP_RESUME;
 1006 	cmd.resume.sid		= sid;
 1007 	cmd.resume.stag		= resp->grpid;
 1008 	switch (resp->code) {
 1009 	case IOMMU_PAGE_RESP_INVALID:
 1010 	case IOMMU_PAGE_RESP_FAILURE:
 1011 		cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
 1012 		break;
 1013 	case IOMMU_PAGE_RESP_SUCCESS:
 1014 		cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
 1015 		break;
 1016 	default:
 1017 		break;
 1018 	}
 1019 
 1020 	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
 1021 	/*
 1022 	 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
 1023 	 * RESUME consumption guarantees that the stalled transaction will be
 1024 	 * terminated... at some point in the future. PRI_RESP is fire and
 1025 	 * forget.
 1026 	 */
 1027 }
 1028 
 1029 /* Context descriptor manipulation functions */
 1030 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
 1031 {
 1032 	struct arm_smmu_cmdq_ent cmd = {
 1033 		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
 1034 			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
 1035 		.tlbi.asid = asid,
 1036 	};
 1037 
 1038 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 1039 }
 1040 
 1041 /*
 1042  * Based on the value of ent report which bits of the STE the HW will access. It
 1043  * would be nice if this was complete according to the spec, but minimally it
 1044  * has to capture the bits this driver uses.
 1045  */
 1046 VISIBLE_IF_KUNIT
 1047 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
 1048 {
 1049 	unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
 1050 
 1051 	used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
 1052 	if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
 1053 		return;
 1054 
 1055 	used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
 1056 
 1057 	/* S1 translates */
 1058 	if (cfg & BIT(0)) {
 1059 		used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
 1060 					    STRTAB_STE_0_S1CTXPTR_MASK |
 1061 					    STRTAB_STE_0_S1CDMAX);
 1062 		used_bits[1] |=
 1063 			cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
 1064 				    STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
 1065 				    STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
 1066 				    STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
 1067 		used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
 1068 
 1069 		/*
 1070 		 * See 13.5 Summary of attribute/permission configuration fields
 1071 		 * for the SHCFG behavior.
 1072 		 */
 1073 		if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
 1074 		    STRTAB_STE_1_S1DSS_BYPASS)
 1075 			used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
 1076 	}
 1077 
 1078 	/* S2 translates */
 1079 	if (cfg & BIT(1)) {
 1080 		used_bits[1] |=
 1081 			cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
 1082 				    STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
 1083 		used_bits[2] |=
 1084 			cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
 1085 				    STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
 1086 				    STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
 1087 				    STRTAB_STE_2_S2R);
 1088 		used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
 1089 	}
 1090 
 1091 	if (cfg == STRTAB_STE_0_CFG_BYPASS)
 1092 		used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
 1093 }
 1094 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
 1095 
 1096 VISIBLE_IF_KUNIT
 1097 void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
 1098 				  __le64 *safe_bits)
 1099 {
 1100 	const __le64 eats_s1chk =
 1101 		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
 1102 	const __le64 eats_trans =
 1103 		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
 1104 
 1105 	/*
 1106 	 * When an STE changes EATS_TRANS, the sequencing code in the attach
 1107 	 * logic already will have the PCI cap for ATS disabled. Thus at this
 1108 	 * moment we can expect that the device will not generate ATS queries
 1109 	 * and so we don't care about the sequencing of EATS. The purpose of
 1110 	 * EATS_TRANS is to protect the system from hostile untrusted devices
 1111 	 * that issue ATS when the PCI config space is disabled. However, if
 1112 	 * EATS_TRANS is being changed, then we must have already trusted the
 1113 	 * device as the EATS_TRANS security block is being disabled.
 1114 	 *
 1115 	 *  Note: now the EATS_TRANS update is moved to the first entry_set().
 1116 	 *  Changing S2S and EATS might transiently result in S2S=1 and EATS=1
 1117 	 *  which is a bad STE (see "5.2 Stream Table Entry"). In such a case,
 1118 	 *  we can't do a hitless update. Also, it should not be added to the
 1119 	 *  safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be
 1120 	 *  effectively an errant 0b00 configuration.
 1121 	 */
 1122 	if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) &&
 1123 	    !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S)))
 1124 		safe_bits[1] |= cpu_to_le64(eats_trans);
 1125 
 1126 	/*
 1127 	 * MEV does not meaningfully impact the operation of the HW, it only
 1128 	 * changes how many fault events are generated, thus we can relax it
 1129 	 * when computing the ordering. The spec notes the device can act like
 1130 	 * MEV=1 anyhow:
 1131 	 *
 1132 	 *  Note: Software must expect, and be able to deal with, coalesced
 1133 	 *  fault records even when MEV == 0.
 1134 	 */
 1135 	safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
 1136 }
 1137 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe);
 1138 
 1139 /*
 1140  * Figure out if we can do a hitless update of entry to become target. Returns a
 1141  * bit mask where 1 indicates that qword needs to be set disruptively.
 1142  * unused_update is an intermediate value of entry that has unused bits set to
 1143  * their new values.
 1144  */
 1145 static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
 1146 				    const __le64 *entry, const __le64 *target,
 1147 				    __le64 *unused_update)
 1148 {
 1149 	__le64 target_used[NUM_ENTRY_QWORDS] = {};
 1150 	__le64 cur_used[NUM_ENTRY_QWORDS] = {};
 1151 	__le64 safe[NUM_ENTRY_QWORDS] = {};
 1152 	u8 used_qword_diff = 0;
 1153 	unsigned int i;
 1154 
 1155 	writer->ops->get_used(entry, cur_used);
 1156 	writer->ops->get_used(target, target_used);
 1157 	if (writer->ops->get_update_safe)
 1158 		writer->ops->get_update_safe(entry, target, safe);
 1159 
 1160 	for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
 1161 		/*
 1162 		 * Safe is only used for bits that are used by both entries,
 1163 		 * otherwise it is sequenced according to the unused entry.
 1164 		 */
 1165 		safe[i] &= target_used[i] & cur_used[i];
 1166 
 1167 		/*
 1168 		 * Check that masks are up to date, the make functions are not
 1169 		 * allowed to set a bit to 1 if the used function doesn't say it
 1170 		 * is used.
 1171 		 */
 1172 		WARN_ON_ONCE(target[i] & ~target_used[i]);
 1173 
 1174 		/* Bits can change because they are not currently being used */
 1175 		cur_used[i] &= ~safe[i];
 1176 		unused_update[i] = (entry[i] & cur_used[i]) |
 1177 				   (target[i] & ~cur_used[i]);
 1178 		/*
 1179 		 * Each bit indicates that a used bit in a qword needs to be
 1180 		 * changed after unused_update is applied.
 1181 		 */
 1182 		if ((unused_update[i] & target_used[i]) != target[i])
 1183 			used_qword_diff |= 1 << i;
 1184 	}
 1185 	return used_qword_diff;
 1186 }
 1187 
 1188 static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
 1189 		      const __le64 *target, unsigned int start,
 1190 		      unsigned int len)
 1191 {
 1192 	bool changed = false;
 1193 	unsigned int i;
 1194 
 1195 	for (i = start; len != 0; len--, i++) {
 1196 		if (entry[i] != target[i]) {
 1197 			WRITE_ONCE(entry[i], target[i]);
 1198 			changed = true;
 1199 		}
 1200 	}
 1201 
 1202 	if (changed)
 1203 		writer->ops->sync(writer);
 1204 }
 1205 
 1206 /*
 1207  * Update the STE/CD to the target configuration. The transition from the
 1208  * current entry to the target entry takes place over multiple steps that
 1209  * attempts to make the transition hitless if possible. This function takes care
 1210  * not to create a situation where the HW can perceive a corrupted entry. HW is
 1211  * only required to have a 64 bit atomicity with stores from the CPU, while
 1212  * entries are many 64 bit values big.
 1213  *
 1214  * The difference between the current value and the target value is analyzed to
 1215  * determine which of three updates are required - disruptive, hitless or no
 1216  * change.
 1217  *
 1218  * In the most general disruptive case we can make any update in three steps:
 1219  *  - Disrupting the entry (V=0)
 1220  *  - Fill now unused qwords, execpt qword 0 which contains V
 1221  *  - Make qword 0 have the final value and valid (V=1) with a single 64
 1222  *    bit store
 1223  *
 1224  * However this disrupts the HW while it is happening. There are several
 1225  * interesting cases where a STE/CD can be updated without disturbing the HW
 1226  * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
 1227  * because the used bits don't intersect. We can detect this by calculating how
 1228  * many 64 bit values need update after adjusting the unused bits and skip the
 1229  * V=0 process. This relies on the IGNORED behavior described in the
 1230  * specification.
 1231  */
 1232 VISIBLE_IF_KUNIT
 1233 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
 1234 			  const __le64 *target)
 1235 {
 1236 	__le64 unused_update[NUM_ENTRY_QWORDS];
 1237 	u8 used_qword_diff;
 1238 
 1239 	/*
 1240 	 * Many of the entry structures have pointers to other structures that
 1241 	 * need to have their updates be visible before any writes of the entry
 1242 	 * happen.
 1243 	 */
 1244 	dma_wmb();
 1245 
 1246 	used_qword_diff =
 1247 		arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
 1248 	if (hweight8(used_qword_diff) == 1) {
 1249 		/*
 1250 		 * Only one qword needs its used bits to be changed. This is a
 1251 		 * hitless update, update all bits the current STE/CD is
 1252 		 * ignoring to their new values, then update a single "critical
 1253 		 * qword" to change the STE/CD and finally 0 out any bits that
 1254 		 * are now unused in the target configuration.
 1255 		 */
 1256 		unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
 1257 
 1258 		/*
 1259 		 * Skip writing unused bits in the critical qword since we'll be
 1260 		 * writing it in the next step anyways. This can save a sync
 1261 		 * when the only change is in that qword.
 1262 		 */
 1263 		unused_update[critical_qword_index] =
 1264 			entry[critical_qword_index];
 1265 		entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
 1266 		entry_set(writer, entry, target, critical_qword_index, 1);
 1267 		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
 1268 	} else if (used_qword_diff) {
 1269 		/*
 1270 		 * At least two qwords need their inuse bits to be changed. This
 1271 		 * requires a breaking update, zero the V bit, write all qwords
 1272 		 * but 0, then set qword 0
 1273 		 */
 1274 		unused_update[0] = 0;
 1275 		entry_set(writer, entry, unused_update, 0, 1);
 1276 		entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
 1277 		entry_set(writer, entry, target, 0, 1);
 1278 	} else {
 1279 		/*
 1280 		 * No inuse bit changed, though safe bits may have changed.
 1281 		 */
 1282 		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
 1283 	}
 1284 }
 1285 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
 1286 
 1287 static void arm_smmu_sync_cd(struct arm_smmu_master *master,
 1288 			     int ssid, bool leaf)
 1289 {
 1290 	size_t i;
 1291 	struct arm_smmu_cmdq_batch cmds;
 1292 	struct arm_smmu_device *smmu = master->smmu;
 1293 	struct arm_smmu_cmdq_ent cmd = {
 1294 		.opcode	= CMDQ_OP_CFGI_CD,
 1295 		.cfgi	= {
 1296 			.ssid	= ssid,
 1297 			.leaf	= leaf,
 1298 		},
 1299 	};
 1300 
 1301 	arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
 1302 	for (i = 0; i < master->num_streams; i++) {
 1303 		cmd.cfgi.sid = master->streams[i].id;
 1304 		arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
 1305 	}
 1306 
 1307 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
 1308 }
 1309 
 1310 static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
 1311 				      dma_addr_t l2ptr_dma)
 1312 {
 1313 	u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
 1314 
 1315 	/* The HW has 64 bit atomicity with stores to the L2 CD table */
 1316 	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
 1317 }
 1318 
 1319 static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
 1320 {
 1321 	return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
 1322 }
 1323 
 1324 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
 1325 					u32 ssid)
 1326 {
 1327 	struct arm_smmu_cdtab_l2 *l2;
 1328 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 1329 
 1330 	if (!arm_smmu_cdtab_allocated(cd_table))
 1331 		return NULL;
 1332 
 1333 	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
 1334 		return &cd_table->linear.table[ssid];
 1335 
 1336 	l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
 1337 	if (!l2)
 1338 		return NULL;
 1339 	return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
 1340 }
 1341 
 1342 static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
 1343 						 u32 ssid)
 1344 {
 1345 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 1346 	struct arm_smmu_device *smmu = master->smmu;
 1347 
 1348 	might_sleep();
 1349 	iommu_group_mutex_assert(master->dev);
 1350 
 1351 	if (!arm_smmu_cdtab_allocated(cd_table)) {
 1352 		if (arm_smmu_alloc_cd_tables(master))
 1353 			return NULL;
 1354 	}
 1355 
 1356 	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
 1357 		unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
 1358 		struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
 1359 
 1360 		if (!*l2ptr) {
 1361 			dma_addr_t l2ptr_dma;
 1362 
 1363 			*l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
 1364 						    &l2ptr_dma, GFP_KERNEL);
 1365 			if (!*l2ptr)
 1366 				return NULL;
 1367 
 1368 			arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
 1369 						  l2ptr_dma);
 1370 			/* An invalid L1CD can be cached */
 1371 			arm_smmu_sync_cd(master, ssid, false);
 1372 		}
 1373 	}
 1374 	return arm_smmu_get_cd_ptr(master, ssid);
 1375 }
 1376 
 1377 struct arm_smmu_cd_writer {
 1378 	struct arm_smmu_entry_writer writer;
 1379 	unsigned int ssid;
 1380 };
 1381 
 1382 VISIBLE_IF_KUNIT
 1383 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
 1384 {
 1385 	used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
 1386 	if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
 1387 		return;
 1388 	memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
 1389 
 1390 	/*
 1391 	 * If EPD0 is set by the make function it means
 1392 	 * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
 1393 	 */
 1394 	if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
 1395 		used_bits[0] &= ~cpu_to_le64(
 1396 			CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
 1397 			CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
 1398 			CTXDESC_CD_0_TCR_SH0);
 1399 		used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
 1400 	}
 1401 }
 1402 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
 1403 
 1404 static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
 1405 {
 1406 	struct arm_smmu_cd_writer *cd_writer =
 1407 		container_of(writer, struct arm_smmu_cd_writer, writer);
 1408 
 1409 	arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
 1410 }
 1411 
 1412 static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
 1413 	.sync = arm_smmu_cd_writer_sync_entry,
 1414 	.get_used = arm_smmu_get_cd_used,
 1415 };
 1416 
 1417 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
 1418 			     struct arm_smmu_cd *cdptr,
 1419 			     const struct arm_smmu_cd *target)
 1420 {
 1421 	bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
 1422 	bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
 1423 	struct arm_smmu_cd_writer cd_writer = {
 1424 		.writer = {
 1425 			.ops = &arm_smmu_cd_writer_ops,
 1426 			.master = master,
 1427 		},
 1428 		.ssid = ssid,
 1429 	};
 1430 
 1431 	if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
 1432 		if (cur_valid)
 1433 			master->cd_table.used_ssids--;
 1434 		else
 1435 			master->cd_table.used_ssids++;
 1436 	}
 1437 
 1438 	arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
 1439 }
 1440 
 1441 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
 1442 			 struct arm_smmu_master *master,
 1443 			 struct arm_smmu_domain *smmu_domain)
 1444 {
 1445 	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
 1446 	const struct io_pgtable_cfg *pgtbl_cfg =
 1447 		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
 1448 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
 1449 		&pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 1450 
 1451 	memset(target, 0, sizeof(*target));
 1452 
 1453 	target->data[0] = cpu_to_le64(
 1454 		FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
 1455 		FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
 1456 		FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
 1457 		FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
 1458 		FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
 1459 #ifdef __BIG_ENDIAN
 1460 		CTXDESC_CD_0_ENDI |
 1461 #endif
 1462 		CTXDESC_CD_0_TCR_EPD1 |
 1463 		CTXDESC_CD_0_V |
 1464 		FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
 1465 		CTXDESC_CD_0_AA64 |
 1466 		(master->stall_enabled ? CTXDESC_CD_0_S : 0) |
 1467 		CTXDESC_CD_0_R |
 1468 		CTXDESC_CD_0_A |
 1469 		CTXDESC_CD_0_ASET |
 1470 		FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
 1471 		);
 1472 
 1473 	/* To enable dirty flag update, set both Access flag and dirty state update */
 1474 	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
 1475 		target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
 1476 					       CTXDESC_CD_0_TCR_HD);
 1477 
 1478 	target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
 1479 				      CTXDESC_CD_1_TTB0_MASK);
 1480 	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
 1481 }
 1482 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
 1483 
 1484 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
 1485 {
 1486 	struct arm_smmu_cd target = {};
 1487 	struct arm_smmu_cd *cdptr;
 1488 
 1489 	if (!arm_smmu_cdtab_allocated(&master->cd_table))
 1490 		return;
 1491 	cdptr = arm_smmu_get_cd_ptr(master, ssid);
 1492 	if (WARN_ON(!cdptr))
 1493 		return;
 1494 	arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
 1495 }
 1496 
 1497 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
 1498 {
 1499 	int ret;
 1500 	size_t l1size;
 1501 	size_t max_contexts;
 1502 	struct arm_smmu_device *smmu = master->smmu;
 1503 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 1504 
 1505 	cd_table->s1cdmax = master->ssid_bits;
 1506 	max_contexts = 1 << cd_table->s1cdmax;
 1507 
 1508 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
 1509 	    max_contexts <= CTXDESC_L2_ENTRIES) {
 1510 		cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
 1511 		cd_table->linear.num_ents = max_contexts;
 1512 
 1513 		l1size = max_contexts * sizeof(struct arm_smmu_cd);
 1514 		cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
 1515 							    &cd_table->cdtab_dma,
 1516 							    GFP_KERNEL);
 1517 		if (!cd_table->linear.table)
 1518 			return -ENOMEM;
 1519 	} else {
 1520 		cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
 1521 		cd_table->l2.num_l1_ents =
 1522 			DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
 1523 
 1524 		cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
 1525 					     sizeof(*cd_table->l2.l2ptrs),
 1526 					     GFP_KERNEL);
 1527 		if (!cd_table->l2.l2ptrs)
 1528 			return -ENOMEM;
 1529 
 1530 		l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
 1531 		cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
 1532 							&cd_table->cdtab_dma,
 1533 							GFP_KERNEL);
 1534 		if (!cd_table->l2.l1tab) {
 1535 			ret = -ENOMEM;
 1536 			goto err_free_l2ptrs;
 1537 		}
 1538 	}
 1539 	return 0;
 1540 
 1541 err_free_l2ptrs:
 1542 	kfree(cd_table->l2.l2ptrs);
 1543 	cd_table->l2.l2ptrs = NULL;
 1544 	return ret;
 1545 }
 1546 
 1547 static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
 1548 {
 1549 	int i;
 1550 	struct arm_smmu_device *smmu = master->smmu;
 1551 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 1552 
 1553 	if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
 1554 		for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
 1555 			if (!cd_table->l2.l2ptrs[i])
 1556 				continue;
 1557 
 1558 			dma_free_coherent(smmu->dev,
 1559 					  sizeof(*cd_table->l2.l2ptrs[i]),
 1560 					  cd_table->l2.l2ptrs[i],
 1561 					  arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
 1562 		}
 1563 		kfree(cd_table->l2.l2ptrs);
 1564 
 1565 		dma_free_coherent(smmu->dev,
 1566 				  cd_table->l2.num_l1_ents *
 1567 					  sizeof(struct arm_smmu_cdtab_l1),
 1568 				  cd_table->l2.l1tab, cd_table->cdtab_dma);
 1569 	} else {
 1570 		dma_free_coherent(smmu->dev,
 1571 				  cd_table->linear.num_ents *
 1572 					  sizeof(struct arm_smmu_cd),
 1573 				  cd_table->linear.table, cd_table->cdtab_dma);
 1574 	}
 1575 }
 1576 
 1577 /* Stream table manipulation functions */
 1578 static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
 1579 					  dma_addr_t l2ptr_dma)
 1580 {
 1581 	u64 val = 0;
 1582 
 1583 	val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
 1584 	val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
 1585 
 1586 	/* The HW has 64 bit atomicity with stores to the L2 STE table */
 1587 	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
 1588 }
 1589 
 1590 struct arm_smmu_ste_writer {
 1591 	struct arm_smmu_entry_writer writer;
 1592 	u32 sid;
 1593 };
 1594 
 1595 static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
 1596 {
 1597 	struct arm_smmu_ste_writer *ste_writer =
 1598 		container_of(writer, struct arm_smmu_ste_writer, writer);
 1599 	struct arm_smmu_cmdq_ent cmd = {
 1600 		.opcode	= CMDQ_OP_CFGI_STE,
 1601 		.cfgi	= {
 1602 			.sid	= ste_writer->sid,
 1603 			.leaf	= true,
 1604 		},
 1605 	};
 1606 
 1607 	arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
 1608 }
 1609 
 1610 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
 1611 	.sync = arm_smmu_ste_writer_sync_entry,
 1612 	.get_used = arm_smmu_get_ste_used,
 1613 	.get_update_safe = arm_smmu_get_ste_update_safe,
 1614 };
 1615 
 1616 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
 1617 			       struct arm_smmu_ste *ste,
 1618 			       const struct arm_smmu_ste *target)
 1619 {
 1620 	struct arm_smmu_device *smmu = master->smmu;
 1621 	struct arm_smmu_ste_writer ste_writer = {
 1622 		.writer = {
 1623 			.ops = &arm_smmu_ste_writer_ops,
 1624 			.master = master,
 1625 		},
 1626 		.sid = sid,
 1627 	};
 1628 
 1629 	arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
 1630 
 1631 	/* It's likely that we'll want to use the new STE soon */
 1632 	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
 1633 		struct arm_smmu_cmdq_ent
 1634 			prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
 1635 					 .prefetch = {
 1636 						 .sid = sid,
 1637 					 } };
 1638 
 1639 		arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
 1640 	}
 1641 }
 1642 
 1643 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
 1644 {
 1645 	memset(target, 0, sizeof(*target));
 1646 	target->data[0] = cpu_to_le64(
 1647 		STRTAB_STE_0_V |
 1648 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
 1649 }
 1650 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
 1651 
 1652 VISIBLE_IF_KUNIT
 1653 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
 1654 			      struct arm_smmu_ste *target)
 1655 {
 1656 	memset(target, 0, sizeof(*target));
 1657 	target->data[0] = cpu_to_le64(
 1658 		STRTAB_STE_0_V |
 1659 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
 1660 
 1661 	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
 1662 		target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
 1663 							 STRTAB_STE_1_SHCFG_INCOMING));
 1664 }
 1665 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
 1666 
 1667 VISIBLE_IF_KUNIT
 1668 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
 1669 			       struct arm_smmu_master *master, bool ats_enabled,
 1670 			       unsigned int s1dss)
 1671 {
 1672 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 1673 	struct arm_smmu_device *smmu = master->smmu;
 1674 
 1675 	memset(target, 0, sizeof(*target));
 1676 	target->data[0] = cpu_to_le64(
 1677 		STRTAB_STE_0_V |
 1678 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
 1679 		FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
 1680 		(cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
 1681 		FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
 1682 
 1683 	target->data[1] = cpu_to_le64(
 1684 		FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
 1685 		FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
 1686 		FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
 1687 		FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
 1688 		((smmu->features & ARM_SMMU_FEAT_STALLS &&
 1689 		  !master->stall_enabled) ?
 1690 			 STRTAB_STE_1_S1STALLD :
 1691 			 0) |
 1692 		FIELD_PREP(STRTAB_STE_1_EATS,
 1693 			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
 1694 
 1695 	if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
 1696 	    s1dss == STRTAB_STE_1_S1DSS_BYPASS)
 1697 		target->data[1] |= cpu_to_le64(FIELD_PREP(
 1698 			STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
 1699 
 1700 	if (smmu->features & ARM_SMMU_FEAT_E2H) {
 1701 		/*
 1702 		 * To support BTM the streamworld needs to match the
 1703 		 * configuration of the CPU so that the ASID broadcasts are
 1704 		 * properly matched. This means either S/NS-EL2-E2H (hypervisor)
 1705 		 * or NS-EL1 (guest). Since an SVA domain can be installed in a
 1706 		 * PASID this should always use a BTM compatible configuration
 1707 		 * if the HW supports it.
 1708 		 */
 1709 		target->data[1] |= cpu_to_le64(
 1710 			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
 1711 	} else {
 1712 		target->data[1] |= cpu_to_le64(
 1713 			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
 1714 
 1715 		/*
 1716 		 * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
 1717 		 * arm_smmu_domain_alloc_id()
 1718 		 */
 1719 		target->data[2] =
 1720 			cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
 1721 	}
 1722 }
 1723 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
 1724 
 1725 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
 1726 				 struct arm_smmu_master *master,
 1727 				 struct arm_smmu_domain *smmu_domain,
 1728 				 bool ats_enabled)
 1729 {
 1730 	struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
 1731 	const struct io_pgtable_cfg *pgtbl_cfg =
 1732 		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
 1733 	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
 1734 		&pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 1735 	u64 vtcr_val;
 1736 	struct arm_smmu_device *smmu = master->smmu;
 1737 
 1738 	memset(target, 0, sizeof(*target));
 1739 	target->data[0] = cpu_to_le64(
 1740 		STRTAB_STE_0_V |
 1741 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
 1742 
 1743 	target->data[1] = cpu_to_le64(
 1744 		FIELD_PREP(STRTAB_STE_1_EATS,
 1745 			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
 1746 
 1747 	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
 1748 		target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
 1749 	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
 1750 		target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
 1751 							  STRTAB_STE_1_SHCFG_INCOMING));
 1752 
 1753 	vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
 1754 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
 1755 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
 1756 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
 1757 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
 1758 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
 1759 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
 1760 	target->data[2] = cpu_to_le64(
 1761 		FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
 1762 		FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
 1763 		STRTAB_STE_2_S2AA64 |
 1764 #ifdef __BIG_ENDIAN
 1765 		STRTAB_STE_2_S2ENDI |
 1766 #endif
 1767 		STRTAB_STE_2_S2PTW |
 1768 		(master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
 1769 		STRTAB_STE_2_S2R);
 1770 
 1771 	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
 1772 				      STRTAB_STE_3_S2TTB_MASK);
 1773 }
 1774 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
 1775 
 1776 /*
 1777  * This can safely directly manipulate the STE memory without a sync sequence
 1778  * because the STE table has not been installed in the SMMU yet.
 1779  */
 1780 static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
 1781 				       unsigned int nent)
 1782 {
 1783 	unsigned int i;
 1784 
 1785 	for (i = 0; i < nent; ++i) {
 1786 		arm_smmu_make_abort_ste(strtab);
 1787 		strtab++;
 1788 	}
 1789 }
 1790 
 1791 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
 1792 {
 1793 	dma_addr_t l2ptr_dma;
 1794 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 1795 	struct arm_smmu_strtab_l2 **l2table;
 1796 
 1797 	l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
 1798 	if (*l2table)
 1799 		return 0;
 1800 
 1801 	*l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
 1802 				       &l2ptr_dma, GFP_KERNEL);
 1803 	if (!*l2table) {
 1804 		dev_err(smmu->dev,
 1805 			"failed to allocate l2 stream table for SID %u\n",
 1806 			sid);
 1807 		return -ENOMEM;
 1808 	}
 1809 
 1810 	arm_smmu_init_initial_stes((*l2table)->stes,
 1811 				   ARRAY_SIZE((*l2table)->stes));
 1812 	arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
 1813 				      l2ptr_dma);
 1814 	return 0;
 1815 }
 1816 
 1817 static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
 1818 {
 1819 	struct arm_smmu_stream *stream_rhs =
 1820 		rb_entry(rhs, struct arm_smmu_stream, node);
 1821 	const u32 *sid_lhs = lhs;
 1822 
 1823 	if (*sid_lhs < stream_rhs->id)
 1824 		return -1;
 1825 	if (*sid_lhs > stream_rhs->id)
 1826 		return 1;
 1827 	return 0;
 1828 }
 1829 
 1830 static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
 1831 				     const struct rb_node *rhs)
 1832 {
 1833 	return arm_smmu_streams_cmp_key(
 1834 		&rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
 1835 }
 1836 
 1837 static struct arm_smmu_master *
 1838 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
 1839 {
 1840 	struct rb_node *node;
 1841 
 1842 	lockdep_assert_held(&smmu->streams_mutex);
 1843 
 1844 	node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
 1845 	if (!node)
 1846 		return NULL;
 1847 	return rb_entry(node, struct arm_smmu_stream, node)->master;
 1848 }
 1849 
 1850 /* IRQ and event handlers */
 1851 static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
 1852 				  struct arm_smmu_event *event)
 1853 {
 1854 	struct arm_smmu_master *master;
 1855 
 1856 	event->id = FIELD_GET(EVTQ_0_ID, raw[0]);
 1857 	event->sid = FIELD_GET(EVTQ_0_SID, raw[0]);
 1858 	event->ssv = FIELD_GET(EVTQ_0_SSV, raw[0]);
 1859 	event->ssid = event->ssv ? FIELD_GET(EVTQ_0_SSID, raw[0]) : IOMMU_NO_PASID;
 1860 	event->privileged = FIELD_GET(EVTQ_1_PnU, raw[1]);
 1861 	event->instruction = FIELD_GET(EVTQ_1_InD, raw[1]);
 1862 	event->s2 = FIELD_GET(EVTQ_1_S2, raw[1]);
 1863 	event->read = FIELD_GET(EVTQ_1_RnW, raw[1]);
 1864 	event->stag = FIELD_GET(EVTQ_1_STAG, raw[1]);
 1865 	event->stall = FIELD_GET(EVTQ_1_STALL, raw[1]);
 1866 	event->class = FIELD_GET(EVTQ_1_CLASS, raw[1]);
 1867 	event->iova = FIELD_GET(EVTQ_2_ADDR, raw[2]);
 1868 	event->ipa = raw[3] & EVTQ_3_IPA;
 1869 	event->fetch_addr = raw[3] & EVTQ_3_FETCH_ADDR;
 1870 	event->ttrnw = FIELD_GET(EVTQ_1_TT_READ, raw[1]);
 1871 	event->class_tt = false;
 1872 	event->dev = NULL;
 1873 
 1874 	if (event->id == EVT_ID_PERMISSION_FAULT)
 1875 		event->class_tt = (event->class == EVTQ_1_CLASS_TT);
 1876 
 1877 	mutex_lock(&smmu->streams_mutex);
 1878 	master = arm_smmu_find_master(smmu, event->sid);
 1879 	if (master)
 1880 		event->dev = get_device(master->dev);
 1881 	mutex_unlock(&smmu->streams_mutex);
 1882 }
 1883 
 1884 static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
 1885 				 struct arm_smmu_event *event)
 1886 {
 1887 	int ret = 0;
 1888 	u32 perm = 0;
 1889 	struct arm_smmu_master *master;
 1890 	struct iopf_fault fault_evt = { };
 1891 	struct iommu_fault *flt = &fault_evt.fault;
 1892 
 1893 	switch (event->id) {
 1894 	case EVT_ID_BAD_STE_CONFIG:
 1895 	case EVT_ID_STREAM_DISABLED_FAULT:
 1896 	case EVT_ID_BAD_SUBSTREAMID_CONFIG:
 1897 	case EVT_ID_BAD_CD_CONFIG:
 1898 	case EVT_ID_TRANSLATION_FAULT:
 1899 	case EVT_ID_ADDR_SIZE_FAULT:
 1900 	case EVT_ID_ACCESS_FAULT:
 1901 	case EVT_ID_PERMISSION_FAULT:
 1902 		break;
 1903 	default:
 1904 		return -EOPNOTSUPP;
 1905 	}
 1906 
 1907 	if (event->stall) {
 1908 		if (event->read)
 1909 			perm |= IOMMU_FAULT_PERM_READ;
 1910 		else
 1911 			perm |= IOMMU_FAULT_PERM_WRITE;
 1912 
 1913 		if (event->instruction)
 1914 			perm |= IOMMU_FAULT_PERM_EXEC;
 1915 
 1916 		if (event->privileged)
 1917 			perm |= IOMMU_FAULT_PERM_PRIV;
 1918 
 1919 		flt->type = IOMMU_FAULT_PAGE_REQ;
 1920 		flt->prm = (struct iommu_fault_page_request){
 1921 			.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
 1922 			.grpid = event->stag,
 1923 			.perm = perm,
 1924 			.addr = event->iova,
 1925 		};
 1926 
 1927 		if (event->ssv) {
 1928 			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
 1929 			flt->prm.pasid = event->ssid;
 1930 		}
 1931 	}
 1932 
 1933 	mutex_lock(&smmu->streams_mutex);
 1934 	master = arm_smmu_find_master(smmu, event->sid);
 1935 	if (!master) {
 1936 		ret = -EINVAL;
 1937 		goto out_unlock;
 1938 	}
 1939 
 1940 	if (event->stall)
 1941 		ret = iommu_report_device_fault(master->dev, &fault_evt);
 1942 	else if (master->vmaster && !event->s2)
 1943 		ret = arm_vmaster_report_event(master->vmaster, evt);
 1944 	else
 1945 		ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
 1946 out_unlock:
 1947 	mutex_unlock(&smmu->streams_mutex);
 1948 	return ret;
 1949 }
 1950 
 1951 static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw,
 1952 				    struct arm_smmu_event *event)
 1953 {
 1954 	int i;
 1955 
 1956 	dev_err(smmu->dev, "event 0x%02x received:\n", event->id);
 1957 
 1958 	for (i = 0; i < EVTQ_ENT_DWORDS; ++i)
 1959 		dev_err(smmu->dev, "\t0x%016llx\n", raw[i]);
 1960 }
 1961 
 1962 #define ARM_SMMU_EVT_KNOWN(e)	((e)->id < ARRAY_SIZE(event_str) && event_str[(e)->id])
 1963 #define ARM_SMMU_LOG_EVT_STR(e) ARM_SMMU_EVT_KNOWN(e) ? event_str[(e)->id] : "UNKNOWN"
 1964 #define ARM_SMMU_LOG_CLIENT(e)	(e)->dev ? dev_name((e)->dev) : "(unassigned sid)"
 1965 
 1966 static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw,
 1967 				struct arm_smmu_event *evt,
 1968 				struct ratelimit_state *rs)
 1969 {
 1970 	if (!__ratelimit(rs))
 1971 		return;
 1972 
 1973 	arm_smmu_dump_raw_event(smmu, raw, evt);
 1974 
 1975 	switch (evt->id) {
 1976 	case EVT_ID_TRANSLATION_FAULT:
 1977 	case EVT_ID_ADDR_SIZE_FAULT:
 1978 	case EVT_ID_ACCESS_FAULT:
 1979 	case EVT_ID_PERMISSION_FAULT:
 1980 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx",
 1981 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
 1982 			evt->sid, evt->ssid, evt->iova, evt->ipa);
 1983 
 1984 		dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x",
 1985 			evt->privileged ? "priv" : "unpriv",
 1986 			evt->instruction ? "inst" : "data",
 1987 			str_read_write(evt->read),
 1988 			evt->s2 ? "s2" : "s1", event_class_str[evt->class],
 1989 			evt->class_tt ? (evt->ttrnw ? " ttd_read" : " ttd_write") : "",
 1990 			evt->stall ? " stall" : "", evt->stag);
 1991 
 1992 		break;
 1993 
 1994 	case EVT_ID_STE_FETCH_FAULT:
 1995 	case EVT_ID_CD_FETCH_FAULT:
 1996 	case EVT_ID_VMS_FETCH_FAULT:
 1997 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx",
 1998 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
 1999 			evt->sid, evt->ssid, evt->fetch_addr);
 2000 
 2001 		break;
 2002 
 2003 	default:
 2004 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x",
 2005 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
 2006 			evt->sid, evt->ssid);
 2007 	}
 2008 }
 2009 
 2010 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 2011 {
 2012 	u64 evt[EVTQ_ENT_DWORDS];
 2013 	struct arm_smmu_event event = {0};
 2014 	struct arm_smmu_device *smmu = dev;
 2015 	struct arm_smmu_queue *q = &smmu->evtq.q;
 2016 	struct arm_smmu_ll_queue *llq = &q->llq;
 2017 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 2018 				      DEFAULT_RATELIMIT_BURST);
 2019 
 2020 	do {
 2021 		while (!queue_remove_raw(q, evt)) {
 2022 			arm_smmu_decode_event(smmu, evt, &event);
 2023 			if (arm_smmu_handle_event(smmu, evt, &event))
 2024 				arm_smmu_dump_event(smmu, evt, &event, &rs);
 2025 
 2026 			put_device(event.dev);
 2027 			cond_resched();
 2028 		}
 2029 
 2030 		/*
 2031 		 * Not much we can do on overflow, so scream and pretend we're
 2032 		 * trying harder.
 2033 		 */
 2034 		if (queue_sync_prod_in(q) == -EOVERFLOW)
 2035 			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
 2036 	} while (!queue_empty(llq));
 2037 
 2038 	/* Sync our overflow flag, as we believe we're up to speed */
 2039 	queue_sync_cons_ovf(q);
 2040 	return IRQ_HANDLED;
 2041 }
 2042 
 2043 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
 2044 {
 2045 	u32 sid, ssid;
 2046 	u16 grpid;
 2047 	bool ssv, last;
 2048 
 2049 	sid = FIELD_GET(PRIQ_0_SID, evt[0]);
 2050 	ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
 2051 	ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
 2052 	last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
 2053 	grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
 2054 
 2055 	dev_info(smmu->dev, "unexpected PRI request received:\n");
 2056 	dev_info(smmu->dev,
 2057 		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
 2058 		 sid, ssid, grpid, last ? "L" : "",
 2059 		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
 2060 		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
 2061 		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
 2062 		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
 2063 		 evt[1] & PRIQ_1_ADDR_MASK);
 2064 
 2065 	if (last) {
 2066 		struct arm_smmu_cmdq_ent cmd = {
 2067 			.opcode			= CMDQ_OP_PRI_RESP,
 2068 			.substream_valid	= ssv,
 2069 			.pri			= {
 2070 				.sid	= sid,
 2071 				.ssid	= ssid,
 2072 				.grpid	= grpid,
 2073 				.resp	= PRI_RESP_DENY,
 2074 			},
 2075 		};
 2076 
 2077 		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
 2078 	}
 2079 }
 2080 
 2081 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
 2082 {
 2083 	struct arm_smmu_device *smmu = dev;
 2084 	struct arm_smmu_queue *q = &smmu->priq.q;
 2085 	struct arm_smmu_ll_queue *llq = &q->llq;
 2086 	u64 evt[PRIQ_ENT_DWORDS];
 2087 
 2088 	do {
 2089 		while (!queue_remove_raw(q, evt))
 2090 			arm_smmu_handle_ppr(smmu, evt);
 2091 
 2092 		if (queue_sync_prod_in(q) == -EOVERFLOW)
 2093 			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
 2094 	} while (!queue_empty(llq));
 2095 
 2096 	/* Sync our overflow flag, as we believe we're up to speed */
 2097 	queue_sync_cons_ovf(q);
 2098 	return IRQ_HANDLED;
 2099 }
 2100 
 2101 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
 2102 
 2103 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
 2104 {
 2105 	u32 gerror, gerrorn, active;
 2106 	struct arm_smmu_device *smmu = dev;
 2107 
 2108 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
 2109 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
 2110 
 2111 	active = gerror ^ gerrorn;
 2112 	if (!(active & GERROR_ERR_MASK))
 2113 		return IRQ_NONE; /* No errors pending */
 2114 
 2115 	dev_warn(smmu->dev,
 2116 		 "unexpected global error reported (0x%08x), this could be serious\n",
 2117 		 active);
 2118 
 2119 	if (active & GERROR_SFM_ERR) {
 2120 		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
 2121 		arm_smmu_device_disable(smmu);
 2122 	}
 2123 
 2124 	if (active & GERROR_MSI_GERROR_ABT_ERR)
 2125 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
 2126 
 2127 	if (active & GERROR_MSI_PRIQ_ABT_ERR)
 2128 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
 2129 
 2130 	if (active & GERROR_MSI_EVTQ_ABT_ERR)
 2131 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
 2132 
 2133 	if (active & GERROR_MSI_CMDQ_ABT_ERR)
 2134 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
 2135 
 2136 	if (active & GERROR_PRIQ_ABT_ERR)
 2137 		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
 2138 
 2139 	if (active & GERROR_EVTQ_ABT_ERR)
 2140 		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
 2141 
 2142 	if (active & GERROR_CMDQ_ERR)
 2143 		arm_smmu_cmdq_skip_err(smmu);
 2144 
 2145 	writel(gerror, smmu->base + ARM_SMMU_GERRORN);
 2146 	return IRQ_HANDLED;
 2147 }
 2148 
 2149 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
 2150 {
 2151 	struct arm_smmu_device *smmu = dev;
 2152 
 2153 	arm_smmu_evtq_thread(irq, dev);
 2154 	if (smmu->features & ARM_SMMU_FEAT_PRI)
 2155 		arm_smmu_priq_thread(irq, dev);
 2156 
 2157 	return IRQ_HANDLED;
 2158 }
 2159 
 2160 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
 2161 {
 2162 	arm_smmu_gerror_handler(irq, dev);
 2163 	return IRQ_WAKE_THREAD;
 2164 }
 2165 
 2166 static void
 2167 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
 2168 			struct arm_smmu_cmdq_ent *cmd)
 2169 {
 2170 	size_t log2_span;
 2171 	size_t span_mask;
 2172 	/* ATC invalidates are always on 4096-bytes pages */
 2173 	size_t inval_grain_shift = 12;
 2174 	unsigned long page_start, page_end;
 2175 
 2176 	/*
 2177 	 * ATS and PASID:
 2178 	 *
 2179 	 * If substream_valid is clear, the PCIe TLP is sent without a PASID
 2180 	 * prefix. In that case all ATC entries within the address range are
 2181 	 * invalidated, including those that were requested with a PASID! There
 2182 	 * is no way to invalidate only entries without PASID.
 2183 	 *
 2184 	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
 2185 	 * traffic), translation requests without PASID create ATC entries
 2186 	 * without PASID, which must be invalidated with substream_valid clear.
 2187 	 * This has the unpleasant side-effect of invalidating all PASID-tagged
 2188 	 * ATC entries within the address range.
 2189 	 */
 2190 	*cmd = (struct arm_smmu_cmdq_ent) {
 2191 		.opcode			= CMDQ_OP_ATC_INV,
 2192 		.substream_valid	= (ssid != IOMMU_NO_PASID),
 2193 		.atc.ssid		= ssid,
 2194 	};
 2195 
 2196 	if (!size) {
 2197 		cmd->atc.size = ATC_INV_SIZE_ALL;
 2198 		return;
 2199 	}
 2200 
 2201 	page_start	= iova >> inval_grain_shift;
 2202 	page_end	= (iova + size - 1) >> inval_grain_shift;
 2203 
 2204 	/*
 2205 	 * In an ATS Invalidate Request, the address must be aligned on the
 2206 	 * range size, which must be a power of two number of page sizes. We
 2207 	 * thus have to choose between grossly over-invalidating the region, or
 2208 	 * splitting the invalidation into multiple commands. For simplicity
 2209 	 * we'll go with the first solution, but should refine it in the future
 2210 	 * if multiple commands are shown to be more efficient.
 2211 	 *
 2212 	 * Find the smallest power of two that covers the range. The most
 2213 	 * significant differing bit between the start and end addresses,
 2214 	 * fls(start ^ end), indicates the required span. For example:
 2215 	 *
 2216 	 * We want to invalidate pages [8; 11]. This is already the ideal range:
 2217 	 *		x = 0b1000 ^ 0b1011 = 0b11
 2218 	 *		span = 1 << fls(x) = 4
 2219 	 *
 2220 	 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
 2221 	 *		x = 0b0111 ^ 0b1010 = 0b1101
 2222 	 *		span = 1 << fls(x) = 16
 2223 	 */
 2224 	log2_span	= fls_long(page_start ^ page_end);
 2225 	span_mask	= (1ULL << log2_span) - 1;
 2226 
 2227 	page_start	&= ~span_mask;
 2228 
 2229 	cmd->atc.addr	= page_start << inval_grain_shift;
 2230 	cmd->atc.size	= log2_span;
 2231 }
 2232 
 2233 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
 2234 				   ioasid_t ssid)
 2235 {
 2236 	int i;
 2237 	struct arm_smmu_cmdq_ent cmd;
 2238 	struct arm_smmu_cmdq_batch cmds;
 2239 
 2240 	arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
 2241 
 2242 	arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
 2243 	for (i = 0; i < master->num_streams; i++) {
 2244 		cmd.atc.sid = master->streams[i].id;
 2245 		arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
 2246 	}
 2247 
 2248 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
 2249 }
 2250 
 2251 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
 2252 			    unsigned long iova, size_t size)
 2253 {
 2254 	struct arm_smmu_master_domain *master_domain;
 2255 	int i;
 2256 	unsigned long flags;
 2257 	struct arm_smmu_cmdq_ent cmd = {
 2258 		.opcode = CMDQ_OP_ATC_INV,
 2259 	};
 2260 	struct arm_smmu_cmdq_batch cmds;
 2261 
 2262 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
 2263 		return 0;
 2264 
 2265 	/*
 2266 	 * Ensure that we've completed prior invalidation of the main TLBs
 2267 	 * before we read 'nr_ats_masters' in case of a concurrent call to
 2268 	 * arm_smmu_enable_ats():
 2269 	 *
 2270 	 *	// unmap()			// arm_smmu_enable_ats()
 2271 	 *	TLBI+SYNC			atomic_inc(&nr_ats_masters);
 2272 	 *	smp_mb();			[...]
 2273 	 *	atomic_read(&nr_ats_masters);	pci_enable_ats() // writel()
 2274 	 *
 2275 	 * Ensures that we always see the incremented 'nr_ats_masters' count if
 2276 	 * ATS was enabled at the PCI device before completion of the TLBI.
 2277 	 */
 2278 	smp_mb();
 2279 	if (!atomic_read(&smmu_domain->nr_ats_masters))
 2280 		return 0;
 2281 
 2282 	arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
 2283 
 2284 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 2285 	list_for_each_entry(master_domain, &smmu_domain->devices,
 2286 			    devices_elm) {
 2287 		struct arm_smmu_master *master = master_domain->master;
 2288 
 2289 		if (!master->ats_enabled)
 2290 			continue;
 2291 
 2292 		if (master_domain->nested_ats_flush) {
 2293 			/*
 2294 			 * If a S2 used as a nesting parent is changed we have
 2295 			 * no option but to completely flush the ATC.
 2296 			 */
 2297 			arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
 2298 		} else {
 2299 			arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
 2300 						&cmd);
 2301 		}
 2302 
 2303 		for (i = 0; i < master->num_streams; i++) {
 2304 			cmd.atc.sid = master->streams[i].id;
 2305 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
 2306 		}
 2307 	}
 2308 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 2309 
 2310 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
 2311 }
 2312 
 2313 /* IO_PGTABLE API */
 2314 static void arm_smmu_tlb_inv_context(void *cookie)
 2315 {
 2316 	struct arm_smmu_domain *smmu_domain = cookie;
 2317 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 2318 	struct arm_smmu_cmdq_ent cmd;
 2319 
 2320 	/*
 2321 	 * NOTE: when io-pgtable is in non-strict mode, we may get here with
 2322 	 * PTEs previously cleared by unmaps on the current CPU not yet visible
 2323 	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
 2324 	 * insertion to guarantee those are observed before the TLBI. Do be
 2325 	 * careful, 007.
 2326 	 */
 2327 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 2328 		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
 2329 	} else {
 2330 		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
 2331 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
 2332 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 2333 	}
 2334 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
 2335 }
 2336 
 2337 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
 2338 				     unsigned long iova, size_t size,
 2339 				     size_t granule,
 2340 				     struct arm_smmu_domain *smmu_domain)
 2341 {
 2342 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 2343 	unsigned long end = iova + size, num_pages = 0, tg = 0;
 2344 	size_t inv_range = granule;
 2345 	struct arm_smmu_cmdq_batch cmds;
 2346 
 2347 	if (!size)
 2348 		return;
 2349 
 2350 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
 2351 		/* Get the leaf page size */
 2352 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
 2353 
 2354 		num_pages = size >> tg;
 2355 
 2356 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
 2357 		cmd->tlbi.tg = (tg - 10) / 2;
 2358 
 2359 		/*
 2360 		 * Determine what level the granule is at. For non-leaf, both
 2361 		 * io-pgtable and SVA pass a nominal last-level granule because
 2362 		 * they don't know what level(s) actually apply, so ignore that
 2363 		 * and leave TTL=0. However for various errata reasons we still
 2364 		 * want to use a range command, so avoid the SVA corner case
 2365 		 * where both scale and num could be 0 as well.
 2366 		 */
 2367 		if (cmd->tlbi.leaf)
 2368 			cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
 2369 		else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
 2370 			num_pages++;
 2371 	}
 2372 
 2373 	arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
 2374 
 2375 	while (iova < end) {
 2376 		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
 2377 			/*
 2378 			 * On each iteration of the loop, the range is 5 bits
 2379 			 * worth of the aligned size remaining.
 2380 			 * The range in pages is:
 2381 			 *
 2382 			 * range = (num_pages & (0x1f << __ffs(num_pages)))
 2383 			 */
 2384 			unsigned long scale, num;
 2385 
 2386 			/* Determine the power of 2 multiple number of pages */
 2387 			scale = __ffs(num_pages);
 2388 			cmd->tlbi.scale = scale;
 2389 
 2390 			/* Determine how many chunks of 2^scale size we have */
 2391 			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
 2392 			cmd->tlbi.num = num - 1;
 2393 
 2394 			/* range is num * 2^scale * pgsize */
 2395 			inv_range = num << (scale + tg);
 2396 
 2397 			/* Clear out the lower order bits for the next iteration */
 2398 			num_pages -= num << scale;
 2399 		}
 2400 
 2401 		cmd->tlbi.addr = iova;
 2402 		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
 2403 		iova += inv_range;
 2404 	}
 2405 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
 2406 }
 2407 
 2408 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
 2409 					  size_t granule, bool leaf,
 2410 					  struct arm_smmu_domain *smmu_domain)
 2411 {
 2412 	struct arm_smmu_cmdq_ent cmd = {
 2413 		.tlbi = {
 2414 			.leaf	= leaf,
 2415 		},
 2416 	};
 2417 
 2418 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 2419 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
 2420 				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
 2421 		cmd.tlbi.asid	= smmu_domain->cd.asid;
 2422 	} else {
 2423 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
 2424 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
 2425 	}
 2426 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
 2427 
 2428 	if (smmu_domain->nest_parent) {
 2429 		/*
 2430 		 * When the S2 domain changes all the nested S1 ASIDs have to be
 2431 		 * flushed too.
 2432 		 */
 2433 		cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
 2434 		arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
 2435 	}
 2436 
 2437 	/*
 2438 	 * Unfortunately, this can't be leaf-only since we may have
 2439 	 * zapped an entire table.
 2440 	 */
 2441 	arm_smmu_atc_inv_domain(smmu_domain, iova, size);
 2442 }
 2443 
 2444 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 2445 				 size_t granule, bool leaf,
 2446 				 struct arm_smmu_domain *smmu_domain)
 2447 {
 2448 	struct arm_smmu_cmdq_ent cmd = {
 2449 		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
 2450 			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
 2451 		.tlbi = {
 2452 			.asid	= asid,
 2453 			.leaf	= leaf,
 2454 		},
 2455 	};
 2456 
 2457 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
 2458 }
 2459 
 2460 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
 2461 					 unsigned long iova, size_t granule,
 2462 					 void *cookie)
 2463 {
 2464 	struct arm_smmu_domain *smmu_domain = cookie;
 2465 	struct iommu_domain *domain = &smmu_domain->domain;
 2466 
 2467 	iommu_iotlb_gather_add_page(domain, gather, iova, granule);
 2468 }
 2469 
 2470 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
 2471 				  size_t granule, void *cookie)
 2472 {
 2473 	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
 2474 }
 2475 
 2476 static const struct iommu_flush_ops arm_smmu_flush_ops = {
 2477 	.tlb_flush_all	= arm_smmu_tlb_inv_context,
 2478 	.tlb_flush_walk = arm_smmu_tlb_inv_walk,
 2479 	.tlb_add_page	= arm_smmu_tlb_inv_page_nosync,
 2480 };
 2481 
 2482 static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
 2483 {
 2484 	u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
 2485 
 2486 	return (smmu->features & features) == features;
 2487 }
 2488 
 2489 /* IOMMU API */
 2490 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
 2491 {
 2492 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 2493 
 2494 	switch (cap) {
 2495 	case IOMMU_CAP_CACHE_COHERENCY:
 2496 		/* Assume that a coherent TCU implies coherent TBUs */
 2497 		return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
 2498 	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
 2499 		return arm_smmu_master_canwbs(master);
 2500 	case IOMMU_CAP_NOEXEC:
 2501 	case IOMMU_CAP_DEFERRED_FLUSH:
 2502 		return true;
 2503 	case IOMMU_CAP_DIRTY_TRACKING:
 2504 		return arm_smmu_dbm_capable(master->smmu);
 2505 	default:
 2506 		return false;
 2507 	}
 2508 }
 2509 
 2510 static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
 2511 {
 2512 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 2513 	struct arm_smmu_master_domain *master_domain;
 2514 	unsigned long flags;
 2515 	bool ret = true;
 2516 
 2517 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 2518 	list_for_each_entry(master_domain, &smmu_domain->devices,
 2519 			    devices_elm) {
 2520 		if (!arm_smmu_master_canwbs(master_domain->master)) {
 2521 			ret = false;
 2522 			break;
 2523 		}
 2524 	}
 2525 	smmu_domain->enforce_cache_coherency = ret;
 2526 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 2527 	return ret;
 2528 }
 2529 
 2530 struct arm_smmu_domain *arm_smmu_domain_alloc(void)
 2531 {
 2532 	struct arm_smmu_domain *smmu_domain;
 2533 
 2534 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 2535 	if (!smmu_domain)
 2536 		return ERR_PTR(-ENOMEM);
 2537 
 2538 	INIT_LIST_HEAD(&smmu_domain->devices);
 2539 	spin_lock_init(&smmu_domain->devices_lock);
 2540 
 2541 	return smmu_domain;
 2542 }
 2543 
 2544 static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
 2545 {
 2546 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 2547 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 2548 
 2549 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 2550 
 2551 	/* Free the ASID or VMID */
 2552 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 2553 		/* Prevent SVA from touching the CD while we're freeing it */
 2554 		mutex_lock(&arm_smmu_asid_lock);
 2555 		xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
 2556 		mutex_unlock(&arm_smmu_asid_lock);
 2557 	} else {
 2558 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
 2559 		if (cfg->vmid)
 2560 			ida_free(&smmu->vmid_map, cfg->vmid);
 2561 	}
 2562 
 2563 	kfree(smmu_domain);
 2564 }
 2565 
 2566 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
 2567 				       struct arm_smmu_domain *smmu_domain)
 2568 {
 2569 	int ret;
 2570 	u32 asid = 0;
 2571 	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
 2572 
 2573 	/* Prevent SVA from modifying the ASID until it is written to the CD */
 2574 	mutex_lock(&arm_smmu_asid_lock);
 2575 	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
 2576 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
 2577 	cd->asid	= (u16)asid;
 2578 	mutex_unlock(&arm_smmu_asid_lock);
 2579 	return ret;
 2580 }
 2581 
 2582 static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
 2583 				       struct arm_smmu_domain *smmu_domain)
 2584 {
 2585 	int vmid;
 2586 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
 2587 
 2588 	/* Reserve VMID 0 for stage-2 bypass STEs */
 2589 	vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
 2590 			       GFP_KERNEL);
 2591 	if (vmid < 0)
 2592 		return vmid;
 2593 
 2594 	cfg->vmid	= (u16)vmid;
 2595 	return 0;
 2596 }
 2597 
 2598 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
 2599 				    struct arm_smmu_device *smmu, u32 flags)
 2600 {
 2601 	int ret;
 2602 	enum io_pgtable_fmt fmt;
 2603 	struct io_pgtable_cfg pgtbl_cfg;
 2604 	struct io_pgtable_ops *pgtbl_ops;
 2605 	int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
 2606 				 struct arm_smmu_domain *smmu_domain);
 2607 	bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
 2608 
 2609 	pgtbl_cfg = (struct io_pgtable_cfg) {
 2610 		.pgsize_bitmap	= smmu->pgsize_bitmap,
 2611 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENCY,
 2612 		.tlb		= &arm_smmu_flush_ops,
 2613 		.iommu_dev	= smmu->dev,
 2614 	};
 2615 
 2616 	switch (smmu_domain->stage) {
 2617 	case ARM_SMMU_DOMAIN_S1: {
 2618 		unsigned long ias = (smmu->features &
 2619 				     ARM_SMMU_FEAT_VAX) ? 52 : 48;
 2620 
 2621 		pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
 2622 		pgtbl_cfg.oas = smmu->ias;
 2623 		if (enable_dirty)
 2624 			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
 2625 		fmt = ARM_64_LPAE_S1;
 2626 		finalise_stage_fn = arm_smmu_domain_finalise_s1;
 2627 		break;
 2628 	}
 2629 	case ARM_SMMU_DOMAIN_S2:
 2630 		if (enable_dirty)
 2631 			return -EOPNOTSUPP;
 2632 		pgtbl_cfg.ias = smmu->ias;
 2633 		pgtbl_cfg.oas = smmu->oas;
 2634 		fmt = ARM_64_LPAE_S2;
 2635 		finalise_stage_fn = arm_smmu_domain_finalise_s2;
 2636 		if ((smmu->features & ARM_SMMU_FEAT_S2FWB) &&
 2637 		    (flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
 2638 			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_S2FWB;
 2639 		break;
 2640 	default:
 2641 		return -EINVAL;
 2642 	}
 2643 
 2644 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 2645 	if (!pgtbl_ops)
 2646 		return -ENOMEM;
 2647 
 2648 	smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 2649 	smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
 2650 	smmu_domain->domain.geometry.force_aperture = true;
 2651 	if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
 2652 		smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
 2653 
 2654 	ret = finalise_stage_fn(smmu, smmu_domain);
 2655 	if (ret < 0) {
 2656 		free_io_pgtable_ops(pgtbl_ops);
 2657 		return ret;
 2658 	}
 2659 
 2660 	smmu_domain->pgtbl_ops = pgtbl_ops;
 2661 	smmu_domain->smmu = smmu;
 2662 	return 0;
 2663 }
 2664 
 2665 static struct arm_smmu_ste *
 2666 arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
 2667 {
 2668 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 2669 
 2670 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
 2671 		/* Two-level walk */
 2672 		return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
 2673 				->stes[arm_smmu_strtab_l2_idx(sid)];
 2674 	} else {
 2675 		/* Simple linear lookup */
 2676 		return &cfg->linear.table[sid];
 2677 	}
 2678 }
 2679 
 2680 void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
 2681 				  const struct arm_smmu_ste *target)
 2682 {
 2683 	int i, j;
 2684 	struct arm_smmu_device *smmu = master->smmu;
 2685 
 2686 	master->cd_table.in_ste =
 2687 		FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
 2688 		STRTAB_STE_0_CFG_S1_TRANS;
 2689 	master->ste_ats_enabled =
 2690 		FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
 2691 		STRTAB_STE_1_EATS_TRANS;
 2692 
 2693 	for (i = 0; i < master->num_streams; ++i) {
 2694 		u32 sid = master->streams[i].id;
 2695 		struct arm_smmu_ste *step =
 2696 			arm_smmu_get_step_for_sid(smmu, sid);
 2697 
 2698 		/* Bridged PCI devices may end up with duplicated IDs */
 2699 		for (j = 0; j < i; j++)
 2700 			if (master->streams[j].id == sid)
 2701 				break;
 2702 		if (j < i)
 2703 			continue;
 2704 
 2705 		arm_smmu_write_ste(master, sid, step, target);
 2706 	}
 2707 }
 2708 
 2709 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
 2710 {
 2711 	struct device *dev = master->dev;
 2712 	struct arm_smmu_device *smmu = master->smmu;
 2713 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 2714 
 2715 	if (!(smmu->features & ARM_SMMU_FEAT_ATS))
 2716 		return false;
 2717 
 2718 	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
 2719 		return false;
 2720 
 2721 	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
 2722 }
 2723 
 2724 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 2725 {
 2726 	size_t stu;
 2727 	struct pci_dev *pdev;
 2728 	struct arm_smmu_device *smmu = master->smmu;
 2729 
 2730 	/* Smallest Translation Unit: log2 of the smallest supported granule */
 2731 	stu = __ffs(smmu->pgsize_bitmap);
 2732 	pdev = to_pci_dev(master->dev);
 2733 
 2734 	/*
 2735 	 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
 2736 	 */
 2737 	arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
 2738 	if (pci_enable_ats(pdev, stu))
 2739 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
 2740 }
 2741 
 2742 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
 2743 {
 2744 	int ret;
 2745 	int features;
 2746 	int num_pasids;
 2747 	struct pci_dev *pdev;
 2748 
 2749 	if (!dev_is_pci(master->dev))
 2750 		return -ENODEV;
 2751 
 2752 	pdev = to_pci_dev(master->dev);
 2753 
 2754 	features = pci_pasid_features(pdev);
 2755 	if (features < 0)
 2756 		return features;
 2757 
 2758 	num_pasids = pci_max_pasids(pdev);
 2759 	if (num_pasids <= 0)
 2760 		return num_pasids;
 2761 
 2762 	ret = pci_enable_pasid(pdev, features);
 2763 	if (ret) {
 2764 		dev_err(&pdev->dev, "Failed to enable PASID\n");
 2765 		return ret;
 2766 	}
 2767 
 2768 	master->ssid_bits = min_t(u8, ilog2(num_pasids),
 2769 				  master->smmu->ssid_bits);
 2770 	return 0;
 2771 }
 2772 
 2773 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
 2774 {
 2775 	struct pci_dev *pdev;
 2776 
 2777 	if (!dev_is_pci(master->dev))
 2778 		return;
 2779 
 2780 	pdev = to_pci_dev(master->dev);
 2781 
 2782 	if (!pdev->pasid_enabled)
 2783 		return;
 2784 
 2785 	master->ssid_bits = 0;
 2786 	pci_disable_pasid(pdev);
 2787 }
 2788 
 2789 static struct arm_smmu_master_domain *
 2790 arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
 2791 			    struct iommu_domain *domain,
 2792 			    struct arm_smmu_master *master,
 2793 			    ioasid_t ssid, bool nested_ats_flush)
 2794 {
 2795 	struct arm_smmu_master_domain *master_domain;
 2796 
 2797 	lockdep_assert_held(&smmu_domain->devices_lock);
 2798 
 2799 	list_for_each_entry(master_domain, &smmu_domain->devices,
 2800 			    devices_elm) {
 2801 		if (master_domain->master == master &&
 2802 		    master_domain->domain == domain &&
 2803 		    master_domain->ssid == ssid &&
 2804 		    master_domain->nested_ats_flush == nested_ats_flush)
 2805 			return master_domain;
 2806 	}
 2807 	return NULL;
 2808 }
 2809 
 2810 /*
 2811  * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
 2812  * structure, otherwise NULL. These domains track attached devices so they can
 2813  * issue invalidations.
 2814  */
 2815 static struct arm_smmu_domain *
 2816 to_smmu_domain_devices(struct iommu_domain *domain)
 2817 {
 2818 	/* The domain can be NULL only when processing the first attach */
 2819 	if (!domain)
 2820 		return NULL;
 2821 	if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
 2822 	    domain->type == IOMMU_DOMAIN_SVA)
 2823 		return to_smmu_domain(domain);
 2824 	if (domain->type == IOMMU_DOMAIN_NESTED)
 2825 		return to_smmu_nested_domain(domain)->vsmmu->s2_parent;
 2826 	return NULL;
 2827 }
 2828 
 2829 static int arm_smmu_enable_iopf(struct arm_smmu_master *master,
 2830 				struct arm_smmu_master_domain *master_domain)
 2831 {
 2832 	int ret;
 2833 
 2834 	iommu_group_mutex_assert(master->dev);
 2835 
 2836 	if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
 2837 		return -EOPNOTSUPP;
 2838 
 2839 	/*
 2840 	 * Drivers for devices supporting PRI or stall require iopf others have
 2841 	 * device-specific fault handlers and don't need IOPF, so this is not a
 2842 	 * failure.
 2843 	 */
 2844 	if (!master->stall_enabled)
 2845 		return 0;
 2846 
 2847 	/* We're not keeping track of SIDs in fault events */
 2848 	if (master->num_streams != 1)
 2849 		return -EOPNOTSUPP;
 2850 
 2851 	if (master->iopf_refcount) {
 2852 		master->iopf_refcount++;
 2853 		master_domain->using_iopf = true;
 2854 		return 0;
 2855 	}
 2856 
 2857 	ret = iopf_queue_add_device(master->smmu->evtq.iopf, master->dev);
 2858 	if (ret)
 2859 		return ret;
 2860 	master->iopf_refcount = 1;
 2861 	master_domain->using_iopf = true;
 2862 	return 0;
 2863 }
 2864 
 2865 static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
 2866 				  struct arm_smmu_master_domain *master_domain)
 2867 {
 2868 	iommu_group_mutex_assert(master->dev);
 2869 
 2870 	if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
 2871 		return;
 2872 
 2873 	if (!master_domain || !master_domain->using_iopf)
 2874 		return;
 2875 
 2876 	master->iopf_refcount--;
 2877 	if (master->iopf_refcount == 0)
 2878 		iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev);
 2879 }
 2880 
 2881 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
 2882 					  struct iommu_domain *domain,
 2883 					  ioasid_t ssid)
 2884 {
 2885 	struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
 2886 	struct arm_smmu_master_domain *master_domain;
 2887 	bool nested_ats_flush = false;
 2888 	unsigned long flags;
 2889 
 2890 	if (!smmu_domain)
 2891 		return;
 2892 
 2893 	if (domain->type == IOMMU_DOMAIN_NESTED)
 2894 		nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
 2895 
 2896 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 2897 	master_domain = arm_smmu_find_master_domain(smmu_domain, domain, master,
 2898 						    ssid, nested_ats_flush);
 2899 	if (master_domain) {
 2900 		list_del(&master_domain->devices_elm);
 2901 		if (master->ats_enabled)
 2902 			atomic_dec(&smmu_domain->nr_ats_masters);
 2903 	}
 2904 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 2905 
 2906 	arm_smmu_disable_iopf(master, master_domain);
 2907 	kfree(master_domain);
 2908 }
 2909 
 2910 /*
 2911  * Start the sequence to attach a domain to a master. The sequence contains three
 2912  * steps:
 2913  *  arm_smmu_attach_prepare()
 2914  *  arm_smmu_install_ste_for_dev()
 2915  *  arm_smmu_attach_commit()
 2916  *
 2917  * If prepare succeeds then the sequence must be completed. The STE installed
 2918  * must set the STE.EATS field according to state.ats_enabled.
 2919  *
 2920  * If the device supports ATS then this determines if EATS should be enabled
 2921  * in the STE, and starts sequencing EATS disable if required.
 2922  *
 2923  * The change of the EATS in the STE and the PCI ATS config space is managed by
 2924  * this sequence to be in the right order so that if PCI ATS is enabled then
 2925  * STE.ETAS is enabled.
 2926  *
 2927  * new_domain can be a non-paging domain. In this case ATS will not be enabled,
 2928  * and invalidations won't be tracked.
 2929  */
 2930 int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
 2931 			    struct iommu_domain *new_domain)
 2932 {
 2933 	struct arm_smmu_master *master = state->master;
 2934 	struct arm_smmu_master_domain *master_domain;
 2935 	struct arm_smmu_domain *smmu_domain =
 2936 		to_smmu_domain_devices(new_domain);
 2937 	unsigned long flags;
 2938 	int ret;
 2939 
 2940 	/*
 2941 	 * arm_smmu_share_asid() must not see two domains pointing to the same
 2942 	 * arm_smmu_master_domain contents otherwise it could randomly write one
 2943 	 * or the other to the CD.
 2944 	 */
 2945 	lockdep_assert_held(&arm_smmu_asid_lock);
 2946 
 2947 	if (smmu_domain || state->cd_needs_ats) {
 2948 		/*
 2949 		 * The SMMU does not support enabling ATS with bypass/abort.
 2950 		 * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
 2951 		 * Translation Requests and Translated transactions are denied
 2952 		 * as though ATS is disabled for the stream (STE.EATS == 0b00),
 2953 		 * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
 2954 		 * (IHI0070Ea 5.2 Stream Table Entry).
 2955 		 *
 2956 		 * However, if we have installed a CD table and are using S1DSS
 2957 		 * then ATS will work in S1DSS bypass. See "13.6.4 Full ATS
 2958 		 * skipping stage 1".
 2959 		 *
 2960 		 * Disable ATS if we are going to create a normal 0b100 bypass
 2961 		 * STE.
 2962 		 */
 2963 		state->ats_enabled = !state->disable_ats &&
 2964 				     arm_smmu_ats_supported(master);
 2965 	}
 2966 
 2967 	if (smmu_domain) {
 2968 		if (new_domain->type == IOMMU_DOMAIN_NESTED) {
 2969 			ret = arm_smmu_attach_prepare_vmaster(
 2970 				state, to_smmu_nested_domain(new_domain));
 2971 			if (ret)
 2972 				return ret;
 2973 		}
 2974 
 2975 		master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
 2976 		if (!master_domain) {
 2977 			ret = -ENOMEM;
 2978 			goto err_free_vmaster;
 2979 		}
 2980 		master_domain->domain = new_domain;
 2981 		master_domain->master = master;
 2982 		master_domain->ssid = state->ssid;
 2983 		if (new_domain->type == IOMMU_DOMAIN_NESTED)
 2984 			master_domain->nested_ats_flush =
 2985 				to_smmu_nested_domain(new_domain)->enable_ats;
 2986 
 2987 		if (new_domain->iopf_handler) {
 2988 			ret = arm_smmu_enable_iopf(master, master_domain);
 2989 			if (ret)
 2990 				goto err_free_master_domain;
 2991 		}
 2992 
 2993 		/*
 2994 		 * During prepare we want the current smmu_domain and new
 2995 		 * smmu_domain to be in the devices list before we change any
 2996 		 * HW. This ensures that both domains will send ATS
 2997 		 * invalidations to the master until we are done.
 2998 		 *
 2999 		 * It is tempting to make this list only track masters that are
 3000 		 * using ATS, but arm_smmu_share_asid() also uses this to change
 3001 		 * the ASID of a domain, unrelated to ATS.
 3002 		 *
 3003 		 * Notice if we are re-attaching the same domain then the list
 3004 		 * will have two identical entries and commit will remove only
 3005 		 * one of them.
 3006 		 */
 3007 		spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 3008 		if (smmu_domain->enforce_cache_coherency &&
 3009 		    !arm_smmu_master_canwbs(master)) {
 3010 			spin_unlock_irqrestore(&smmu_domain->devices_lock,
 3011 					       flags);
 3012 			ret = -EINVAL;
 3013 			goto err_iopf;
 3014 		}
 3015 
 3016 		if (state->ats_enabled)
 3017 			atomic_inc(&smmu_domain->nr_ats_masters);
 3018 		list_add(&master_domain->devices_elm, &smmu_domain->devices);
 3019 		spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 3020 	}
 3021 
 3022 	if (!state->ats_enabled && master->ats_enabled) {
 3023 		pci_disable_ats(to_pci_dev(master->dev));
 3024 		/*
 3025 		 * This is probably overkill, but the config write for disabling
 3026 		 * ATS should complete before the STE is configured to generate
 3027 		 * UR to avoid AER noise.
 3028 		 */
 3029 		wmb();
 3030 	}
 3031 	return 0;
 3032 
 3033 err_iopf:
 3034 	arm_smmu_disable_iopf(master, master_domain);
 3035 err_free_master_domain:
 3036 	kfree(master_domain);
 3037 err_free_vmaster:
 3038 	kfree(state->vmaster);
 3039 	return ret;
 3040 }
 3041 
 3042 /*
 3043  * Commit is done after the STE/CD are configured with the EATS setting. It
 3044  * completes synchronizing the PCI device's ATC and finishes manipulating the
 3045  * smmu_domain->devices list.
 3046  */
 3047 void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
 3048 {
 3049 	struct arm_smmu_master *master = state->master;
 3050 
 3051 	lockdep_assert_held(&arm_smmu_asid_lock);
 3052 
 3053 	arm_smmu_attach_commit_vmaster(state);
 3054 
 3055 	if (state->ats_enabled && !master->ats_enabled) {
 3056 		arm_smmu_enable_ats(master);
 3057 	} else if (state->ats_enabled && master->ats_enabled) {
 3058 		/*
 3059 		 * The translation has changed, flush the ATC. At this point the
 3060 		 * SMMU is translating for the new domain and both the old&new
 3061 		 * domain will issue invalidations.
 3062 		 */
 3063 		arm_smmu_atc_inv_master(master, state->ssid);
 3064 	} else if (!state->ats_enabled && master->ats_enabled) {
 3065 		/* ATS is being switched off, invalidate the entire ATC */
 3066 		arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
 3067 	}
 3068 
 3069 	arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
 3070 	master->ats_enabled = state->ats_enabled;
 3071 }
 3072 
 3073 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 3074 {
 3075 	int ret = 0;
 3076 	struct arm_smmu_ste target;
 3077 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 3078 	struct arm_smmu_device *smmu;
 3079 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3080 	struct arm_smmu_attach_state state = {
 3081 		.old_domain = iommu_get_domain_for_dev(dev),
 3082 		.ssid = IOMMU_NO_PASID,
 3083 	};
 3084 	struct arm_smmu_master *master;
 3085 	struct arm_smmu_cd *cdptr;
 3086 
 3087 	if (!fwspec)
 3088 		return -ENOENT;
 3089 
 3090 	state.master = master = dev_iommu_priv_get(dev);
 3091 	smmu = master->smmu;
 3092 
 3093 	if (smmu_domain->smmu != smmu)
 3094 		return -EINVAL;
 3095 
 3096 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 3097 		cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
 3098 		if (!cdptr)
 3099 			return -ENOMEM;
 3100 	} else if (arm_smmu_ssids_in_use(&master->cd_table))
 3101 		return -EBUSY;
 3102 
 3103 	/*
 3104 	 * Prevent arm_smmu_share_asid() from trying to change the ASID
 3105 	 * of either the old or new domain while we are working on it.
 3106 	 * This allows the STE and the smmu_domain->devices list to
 3107 	 * be inconsistent during this routine.
 3108 	 */
 3109 	mutex_lock(&arm_smmu_asid_lock);
 3110 
 3111 	ret = arm_smmu_attach_prepare(&state, domain);
 3112 	if (ret) {
 3113 		mutex_unlock(&arm_smmu_asid_lock);
 3114 		return ret;
 3115 	}
 3116 
 3117 	switch (smmu_domain->stage) {
 3118 	case ARM_SMMU_DOMAIN_S1: {
 3119 		struct arm_smmu_cd target_cd;
 3120 
 3121 		arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
 3122 		arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
 3123 					&target_cd);
 3124 		arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
 3125 					  STRTAB_STE_1_S1DSS_SSID0);
 3126 		arm_smmu_install_ste_for_dev(master, &target);
 3127 		break;
 3128 	}
 3129 	case ARM_SMMU_DOMAIN_S2:
 3130 		arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
 3131 					    state.ats_enabled);
 3132 		arm_smmu_install_ste_for_dev(master, &target);
 3133 		arm_smmu_clear_cd(master, IOMMU_NO_PASID);
 3134 		break;
 3135 	}
 3136 
 3137 	arm_smmu_attach_commit(&state);
 3138 	mutex_unlock(&arm_smmu_asid_lock);
 3139 	return 0;
 3140 }
 3141 
 3142 static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
 3143 				     struct device *dev, ioasid_t id,
 3144 				     struct iommu_domain *old)
 3145 {
 3146 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3147 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3148 	struct arm_smmu_device *smmu = master->smmu;
 3149 	struct arm_smmu_cd target_cd;
 3150 
 3151 	if (smmu_domain->smmu != smmu)
 3152 		return -EINVAL;
 3153 
 3154 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
 3155 		return -EINVAL;
 3156 
 3157 	/*
 3158 	 * We can read cd.asid outside the lock because arm_smmu_set_pasid()
 3159 	 * will fix it
 3160 	 */
 3161 	arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
 3162 	return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
 3163 				  &target_cd, old);
 3164 }
 3165 
 3166 static void arm_smmu_update_ste(struct arm_smmu_master *master,
 3167 				struct iommu_domain *sid_domain,
 3168 				bool ats_enabled)
 3169 {
 3170 	unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
 3171 	struct arm_smmu_ste ste;
 3172 
 3173 	if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
 3174 		return;
 3175 
 3176 	if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
 3177 		s1dss = STRTAB_STE_1_S1DSS_BYPASS;
 3178 	else
 3179 		WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
 3180 
 3181 	/*
 3182 	 * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
 3183 	 * using s1dss if necessary. If the cd_table is already installed then
 3184 	 * the S1DSS is correct and this will just update the EATS. Otherwise it
 3185 	 * installs the entire thing. This will be hitless.
 3186 	 */
 3187 	arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
 3188 	arm_smmu_install_ste_for_dev(master, &ste);
 3189 }
 3190 
 3191 int arm_smmu_set_pasid(struct arm_smmu_master *master,
 3192 		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
 3193 		       struct arm_smmu_cd *cd, struct iommu_domain *old)
 3194 {
 3195 	struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
 3196 	struct arm_smmu_attach_state state = {
 3197 		.master = master,
 3198 		.ssid = pasid,
 3199 		.old_domain = old,
 3200 	};
 3201 	struct arm_smmu_cd *cdptr;
 3202 	int ret;
 3203 
 3204 	/* The core code validates pasid */
 3205 
 3206 	if (smmu_domain->smmu != master->smmu)
 3207 		return -EINVAL;
 3208 
 3209 	if (!master->cd_table.in_ste &&
 3210 	    sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
 3211 	    sid_domain->type != IOMMU_DOMAIN_BLOCKED)
 3212 		return -EINVAL;
 3213 
 3214 	cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
 3215 	if (!cdptr)
 3216 		return -ENOMEM;
 3217 
 3218 	mutex_lock(&arm_smmu_asid_lock);
 3219 	ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
 3220 	if (ret)
 3221 		goto out_unlock;
 3222 
 3223 	/*
 3224 	 * We don't want to obtain to the asid_lock too early, so fix up the
 3225 	 * caller set ASID under the lock in case it changed.
 3226 	 */
 3227 	cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
 3228 	cd->data[0] |= cpu_to_le64(
 3229 		FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
 3230 
 3231 	arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
 3232 	arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
 3233 
 3234 	arm_smmu_attach_commit(&state);
 3235 
 3236 out_unlock:
 3237 	mutex_unlock(&arm_smmu_asid_lock);
 3238 	return ret;
 3239 }
 3240 
 3241 static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
 3242 					   struct device *dev, ioasid_t pasid,
 3243 					   struct iommu_domain *old_domain)
 3244 {
 3245 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
 3246 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3247 
 3248 	mutex_lock(&arm_smmu_asid_lock);
 3249 	arm_smmu_clear_cd(master, pasid);
 3250 	if (master->ats_enabled)
 3251 		arm_smmu_atc_inv_master(master, pasid);
 3252 	arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
 3253 	mutex_unlock(&arm_smmu_asid_lock);
 3254 
 3255 	/*
 3256 	 * When the last user of the CD table goes away downgrade the STE back
 3257 	 * to a non-cd_table one.
 3258 	 */
 3259 	if (!arm_smmu_ssids_in_use(&master->cd_table)) {
 3260 		struct iommu_domain *sid_domain =
 3261 			iommu_get_domain_for_dev(master->dev);
 3262 
 3263 		if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
 3264 		    sid_domain->type == IOMMU_DOMAIN_BLOCKED)
 3265 			sid_domain->ops->attach_dev(sid_domain, dev);
 3266 	}
 3267 	return 0;
 3268 }
 3269 
 3270 static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
 3271 				    struct device *dev,
 3272 				    struct arm_smmu_ste *ste,
 3273 				    unsigned int s1dss)
 3274 {
 3275 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3276 	struct arm_smmu_attach_state state = {
 3277 		.master = master,
 3278 		.old_domain = iommu_get_domain_for_dev(dev),
 3279 		.ssid = IOMMU_NO_PASID,
 3280 	};
 3281 
 3282 	/*
 3283 	 * Do not allow any ASID to be changed while are working on the STE,
 3284 	 * otherwise we could miss invalidations.
 3285 	 */
 3286 	mutex_lock(&arm_smmu_asid_lock);
 3287 
 3288 	/*
 3289 	 * If the CD table is not in use we can use the provided STE, otherwise
 3290 	 * we use a cdtable STE with the provided S1DSS.
 3291 	 */
 3292 	if (arm_smmu_ssids_in_use(&master->cd_table)) {
 3293 		/*
 3294 		 * If a CD table has to be present then we need to run with ATS
 3295 		 * on because we have to assume a PASID is using ATS. For
 3296 		 * IDENTITY this will setup things so that S1DSS=bypass which
 3297 		 * follows the explanation in "13.6.4 Full ATS skipping stage 1"
 3298 		 * and allows for ATS on the RID to work.
 3299 		 */
 3300 		state.cd_needs_ats = true;
 3301 		arm_smmu_attach_prepare(&state, domain);
 3302 		arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
 3303 	} else {
 3304 		arm_smmu_attach_prepare(&state, domain);
 3305 	}
 3306 	arm_smmu_install_ste_for_dev(master, ste);
 3307 	arm_smmu_attach_commit(&state);
 3308 	mutex_unlock(&arm_smmu_asid_lock);
 3309 
 3310 	/*
 3311 	 * This has to be done after removing the master from the
 3312 	 * arm_smmu_domain->devices to avoid races updating the same context
 3313 	 * descriptor from arm_smmu_share_asid().
 3314 	 */
 3315 	arm_smmu_clear_cd(master, IOMMU_NO_PASID);
 3316 }
 3317 
 3318 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
 3319 					struct device *dev)
 3320 {
 3321 	struct arm_smmu_ste ste;
 3322 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3323 
 3324 	arm_smmu_master_clear_vmaster(master);
 3325 	arm_smmu_make_bypass_ste(master->smmu, &ste);
 3326 	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
 3327 	return 0;
 3328 }
 3329 
 3330 static const struct iommu_domain_ops arm_smmu_identity_ops = {
 3331 	.attach_dev = arm_smmu_attach_dev_identity,
 3332 };
 3333 
 3334 static struct iommu_domain arm_smmu_identity_domain = {
 3335 	.type = IOMMU_DOMAIN_IDENTITY,
 3336 	.ops = &arm_smmu_identity_ops,
 3337 };
 3338 
 3339 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
 3340 					struct device *dev)
 3341 {
 3342 	struct arm_smmu_ste ste;
 3343 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3344 
 3345 	arm_smmu_master_clear_vmaster(master);
 3346 	arm_smmu_make_abort_ste(&ste);
 3347 	arm_smmu_attach_dev_ste(domain, dev, &ste,
 3348 				STRTAB_STE_1_S1DSS_TERMINATE);
 3349 	return 0;
 3350 }
 3351 
 3352 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
 3353 	.attach_dev = arm_smmu_attach_dev_blocked,
 3354 	.set_dev_pasid = arm_smmu_blocking_set_dev_pasid,
 3355 };
 3356 
 3357 static struct iommu_domain arm_smmu_blocked_domain = {
 3358 	.type = IOMMU_DOMAIN_BLOCKED,
 3359 	.ops = &arm_smmu_blocked_ops,
 3360 };
 3361 
 3362 static struct iommu_domain *
 3363 arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
 3364 				   const struct iommu_user_data *user_data)
 3365 {
 3366 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3367 	struct arm_smmu_device *smmu = master->smmu;
 3368 	const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
 3369 				 IOMMU_HWPT_ALLOC_PASID |
 3370 				 IOMMU_HWPT_ALLOC_NEST_PARENT;
 3371 	struct arm_smmu_domain *smmu_domain;
 3372 	int ret;
 3373 
 3374 	if (flags & ~PAGING_FLAGS)
 3375 		return ERR_PTR(-EOPNOTSUPP);
 3376 	if (user_data)
 3377 		return ERR_PTR(-EOPNOTSUPP);
 3378 
 3379 	smmu_domain = arm_smmu_domain_alloc();
 3380 	if (IS_ERR(smmu_domain))
 3381 		return ERR_CAST(smmu_domain);
 3382 
 3383 	switch (flags) {
 3384 	case 0:
 3385 		/* Prefer S1 if available */
 3386 		if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
 3387 			smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 3388 		else
 3389 			smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 3390 		break;
 3391 	case IOMMU_HWPT_ALLOC_NEST_PARENT:
 3392 		if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) {
 3393 			ret = -EOPNOTSUPP;
 3394 			goto err_free;
 3395 		}
 3396 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 3397 		smmu_domain->nest_parent = true;
 3398 		break;
 3399 	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
 3400 	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_PASID:
 3401 	case IOMMU_HWPT_ALLOC_PASID:
 3402 		if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) {
 3403 			ret = -EOPNOTSUPP;
 3404 			goto err_free;
 3405 		}
 3406 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 3407 		break;
 3408 	default:
 3409 		ret = -EOPNOTSUPP;
 3410 		goto err_free;
 3411 	}
 3412 
 3413 	smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
 3414 	smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
 3415 	ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags);
 3416 	if (ret)
 3417 		goto err_free;
 3418 	return &smmu_domain->domain;
 3419 
 3420 err_free:
 3421 	kfree(smmu_domain);
 3422 	return ERR_PTR(ret);
 3423 }
 3424 
 3425 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
 3426 			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
 3427 			      int prot, gfp_t gfp, size_t *mapped)
 3428 {
 3429 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
 3430 
 3431 	if (!ops)
 3432 		return -ENODEV;
 3433 
 3434 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
 3435 }
 3436 
 3437 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
 3438 				   size_t pgsize, size_t pgcount,
 3439 				   struct iommu_iotlb_gather *gather)
 3440 {
 3441 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3442 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
 3443 
 3444 	if (!ops)
 3445 		return 0;
 3446 
 3447 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
 3448 }
 3449 
 3450 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
 3451 {
 3452 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3453 
 3454 	if (smmu_domain->smmu)
 3455 		arm_smmu_tlb_inv_context(smmu_domain);
 3456 }
 3457 
 3458 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
 3459 				struct iommu_iotlb_gather *gather)
 3460 {
 3461 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3462 
 3463 	if (!gather->pgsize)
 3464 		return;
 3465 
 3466 	arm_smmu_tlb_inv_range_domain(gather->start,
 3467 				      gather->end - gather->start + 1,
 3468 				      gather->pgsize, true, smmu_domain);
 3469 }
 3470 
 3471 static phys_addr_t
 3472 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
 3473 {
 3474 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
 3475 
 3476 	if (!ops)
 3477 		return 0;
 3478 
 3479 	return ops->iova_to_phys(ops, iova);
 3480 }
 3481 
 3482 static struct platform_driver arm_smmu_driver;
 3483 
 3484 static
 3485 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
 3486 {
 3487 	struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
 3488 
 3489 	put_device(dev);
 3490 	return dev ? dev_get_drvdata(dev) : NULL;
 3491 }
 3492 
 3493 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
 3494 {
 3495 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
 3496 		return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
 3497 	return sid < smmu->strtab_cfg.linear.num_ents;
 3498 }
 3499 
 3500 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
 3501 {
 3502 	/* Check the SIDs are in range of the SMMU and our stream table */
 3503 	if (!arm_smmu_sid_in_range(smmu, sid))
 3504 		return -ERANGE;
 3505 
 3506 	/* Ensure l2 strtab is initialised */
 3507 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
 3508 		return arm_smmu_init_l2_strtab(smmu, sid);
 3509 
 3510 	return 0;
 3511 }
 3512 
 3513 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
 3514 				  struct arm_smmu_master *master)
 3515 {
 3516 	int i;
 3517 	int ret = 0;
 3518 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
 3519 
 3520 	master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
 3521 				  GFP_KERNEL);
 3522 	if (!master->streams)
 3523 		return -ENOMEM;
 3524 	master->num_streams = fwspec->num_ids;
 3525 
 3526 	mutex_lock(&smmu->streams_mutex);
 3527 	for (i = 0; i < fwspec->num_ids; i++) {
 3528 		struct arm_smmu_stream *new_stream = &master->streams[i];
 3529 		struct rb_node *existing;
 3530 		u32 sid = fwspec->ids[i];
 3531 
 3532 		new_stream->id = sid;
 3533 		new_stream->master = master;
 3534 
 3535 		ret = arm_smmu_init_sid_strtab(smmu, sid);
 3536 		if (ret)
 3537 			break;
 3538 
 3539 		/* Insert into SID tree */
 3540 		existing = rb_find_add(&new_stream->node, &smmu->streams,
 3541 				       arm_smmu_streams_cmp_node);
 3542 		if (existing) {
 3543 			struct arm_smmu_master *existing_master =
 3544 				rb_entry(existing, struct arm_smmu_stream, node)
 3545 					->master;
 3546 
 3547 			/* Bridged PCI devices may end up with duplicated IDs */
 3548 			if (existing_master == master)
 3549 				continue;
 3550 
 3551 			dev_warn(master->dev,
 3552 				 "Aliasing StreamID 0x%x (from %s) unsupported, expect DMA to be broken\n",
 3553 				 sid, dev_name(existing_master->dev));
 3554 			ret = -ENODEV;
 3555 			break;
 3556 		}
 3557 	}
 3558 
 3559 	if (ret) {
 3560 		for (i--; i >= 0; i--)
 3561 			rb_erase(&master->streams[i].node, &smmu->streams);
 3562 		kfree(master->streams);
 3563 	}
 3564 	mutex_unlock(&smmu->streams_mutex);
 3565 
 3566 	return ret;
 3567 }
 3568 
 3569 static void arm_smmu_remove_master(struct arm_smmu_master *master)
 3570 {
 3571 	int i;
 3572 	struct arm_smmu_device *smmu = master->smmu;
 3573 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
 3574 
 3575 	if (!smmu || !master->streams)
 3576 		return;
 3577 
 3578 	mutex_lock(&smmu->streams_mutex);
 3579 	for (i = 0; i < fwspec->num_ids; i++)
 3580 		rb_erase(&master->streams[i].node, &smmu->streams);
 3581 	mutex_unlock(&smmu->streams_mutex);
 3582 
 3583 	kfree(master->streams);
 3584 }
 3585 
 3586 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 3587 {
 3588 	int ret;
 3589 	struct arm_smmu_device *smmu;
 3590 	struct arm_smmu_master *master;
 3591 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 3592 
 3593 	if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
 3594 		return ERR_PTR(-EBUSY);
 3595 
 3596 	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
 3597 	if (!smmu)
 3598 		return ERR_PTR(-ENODEV);
 3599 
 3600 	master = kzalloc(sizeof(*master), GFP_KERNEL);
 3601 	if (!master)
 3602 		return ERR_PTR(-ENOMEM);
 3603 
 3604 	master->dev = dev;
 3605 	master->smmu = smmu;
 3606 	dev_iommu_priv_set(dev, master);
 3607 
 3608 	ret = arm_smmu_insert_master(smmu, master);
 3609 	if (ret)
 3610 		goto err_free_master;
 3611 
 3612 	device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
 3613 	master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
 3614 
 3615 	/*
 3616 	 * Note that PASID must be enabled before, and disabled after ATS:
 3617 	 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
 3618 	 *
 3619 	 *   Behavior is undefined if this bit is Set and the value of the PASID
 3620 	 *   Enable, Execute Requested Enable, or Privileged Mode Requested bits
 3621 	 *   are changed.
 3622 	 */
 3623 	arm_smmu_enable_pasid(master);
 3624 
 3625 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
 3626 		master->ssid_bits = min_t(u8, master->ssid_bits,
 3627 					  CTXDESC_LINEAR_CDMAX);
 3628 
 3629 	if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
 3630 	     device_property_read_bool(dev, "dma-can-stall")) ||
 3631 	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
 3632 		master->stall_enabled = true;
 3633 
 3634 	if (dev_is_pci(dev)) {
 3635 		unsigned int stu = __ffs(smmu->pgsize_bitmap);
 3636 
 3637 		pci_prepare_ats(to_pci_dev(dev), stu);
 3638 	}
 3639 
 3640 	return &smmu->iommu;
 3641 
 3642 err_free_master:
 3643 	kfree(master);
 3644 	return ERR_PTR(ret);
 3645 }
 3646 
 3647 static void arm_smmu_release_device(struct device *dev)
 3648 {
 3649 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 3650 
 3651 	WARN_ON(master->iopf_refcount);
 3652 
 3653 	/* Put the STE back to what arm_smmu_init_strtab() sets */
 3654 	if (dev->iommu->require_direct)
 3655 		arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev);
 3656 	else
 3657 		arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev);
 3658 
 3659 	arm_smmu_disable_pasid(master);
 3660 	arm_smmu_remove_master(master);
 3661 	if (arm_smmu_cdtab_allocated(&master->cd_table))
 3662 		arm_smmu_free_cd_tables(master);
 3663 	kfree(master);
 3664 }
 3665 
 3666 static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
 3667 					 unsigned long iova, size_t size,
 3668 					 unsigned long flags,
 3669 					 struct iommu_dirty_bitmap *dirty)
 3670 {
 3671 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 3672 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
 3673 
 3674 	return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
 3675 }
 3676 
 3677 static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
 3678 				       bool enabled)
 3679 {
 3680 	/*
 3681 	 * Always enabled and the dirty bitmap is cleared prior to
 3682 	 * set_dirty_tracking().
 3683 	 */
 3684 	return 0;
 3685 }
 3686 
 3687 static struct iommu_group *arm_smmu_device_group(struct device *dev)
 3688 {
 3689 	struct iommu_group *group;
 3690 
 3691 	/*
 3692 	 * We don't support devices sharing stream IDs other than PCI RID
 3693 	 * aliases, since the necessary ID-to-device lookup becomes rather
 3694 	 * impractical given a potential sparse 32-bit stream ID space.
 3695 	 */
 3696 	if (dev_is_pci(dev))
 3697 		group = pci_device_group(dev);
 3698 	else
 3699 		group = generic_device_group(dev);
 3700 
 3701 	return group;
 3702 }
 3703 
 3704 static int arm_smmu_of_xlate(struct device *dev,
 3705 			     const struct of_phandle_args *args)
 3706 {
 3707 	return iommu_fwspec_add_ids(dev, args->args, 1);
 3708 }
 3709 
 3710 static void arm_smmu_get_resv_regions(struct device *dev,
 3711 				      struct list_head *head)
 3712 {
 3713 	struct iommu_resv_region *region;
 3714 	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
 3715 
 3716 	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
 3717 					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
 3718 	if (!region)
 3719 		return;
 3720 
 3721 	list_add_tail(&region->list, head);
 3722 
 3723 	iommu_dma_get_resv_regions(dev, head);
 3724 }
 3725 
 3726 /*
 3727  * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
 3728  * PCIe link and save the data to memory by DMA. The hardware is restricted to
 3729  * use identity mapping only.
 3730  */
 3731 #define IS_HISI_PTT_DEVICE(pdev)	((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
 3732 					 (pdev)->device == 0xa12e)
 3733 
 3734 static int arm_smmu_def_domain_type(struct device *dev)
 3735 {
 3736 	if (dev_is_pci(dev)) {
 3737 		struct pci_dev *pdev = to_pci_dev(dev);
 3738 
 3739 		if (IS_HISI_PTT_DEVICE(pdev))
 3740 			return IOMMU_DOMAIN_IDENTITY;
 3741 	}
 3742 
 3743 	return 0;
 3744 }
 3745 
 3746 static const struct iommu_ops arm_smmu_ops = {
 3747 	.identity_domain	= &arm_smmu_identity_domain,
 3748 	.blocked_domain		= &arm_smmu_blocked_domain,
 3749 	.capable		= arm_smmu_capable,
 3750 	.hw_info		= arm_smmu_hw_info,
 3751 	.domain_alloc_sva       = arm_smmu_sva_domain_alloc,
 3752 	.domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags,
 3753 	.probe_device		= arm_smmu_probe_device,
 3754 	.release_device		= arm_smmu_release_device,
 3755 	.device_group		= arm_smmu_device_group,
 3756 	.of_xlate		= arm_smmu_of_xlate,
 3757 	.get_resv_regions	= arm_smmu_get_resv_regions,
 3758 	.page_response		= arm_smmu_page_response,
 3759 	.def_domain_type	= arm_smmu_def_domain_type,
 3760 	.get_viommu_size	= arm_smmu_get_viommu_size,
 3761 	.viommu_init		= arm_vsmmu_init,
 3762 	.user_pasid_table	= 1,
 3763 	.owner			= THIS_MODULE,
 3764 	.default_domain_ops = &(const struct iommu_domain_ops) {
 3765 		.attach_dev		= arm_smmu_attach_dev,
 3766 		.enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
 3767 		.set_dev_pasid		= arm_smmu_s1_set_dev_pasid,
 3768 		.map_pages		= arm_smmu_map_pages,
 3769 		.unmap_pages		= arm_smmu_unmap_pages,
 3770 		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
 3771 		.iotlb_sync		= arm_smmu_iotlb_sync,
 3772 		.iova_to_phys		= arm_smmu_iova_to_phys,
 3773 		.free			= arm_smmu_domain_free_paging,
 3774 	}
 3775 };
 3776 
 3777 static struct iommu_dirty_ops arm_smmu_dirty_ops = {
 3778 	.read_and_clear_dirty	= arm_smmu_read_and_clear_dirty,
 3779 	.set_dirty_tracking     = arm_smmu_set_dirty_tracking,
 3780 };
 3781 
 3782 /* Probing and initialisation functions */
 3783 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
 3784 			    struct arm_smmu_queue *q, void __iomem *page,
 3785 			    unsigned long prod_off, unsigned long cons_off,
 3786 			    size_t dwords, const char *name)
 3787 {
 3788 	size_t qsz;
 3789 
 3790 	do {
 3791 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
 3792 		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
 3793 					      GFP_KERNEL);
 3794 		if (q->base || qsz < PAGE_SIZE)
 3795 			break;
 3796 
 3797 		q->llq.max_n_shift--;
 3798 	} while (1);
 3799 
 3800 	if (!q->base) {
 3801 		dev_err(smmu->dev,
 3802 			"failed to allocate queue (0x%zx bytes) for %s\n",
 3803 			qsz, name);
 3804 		return -ENOMEM;
 3805 	}
 3806 
 3807 	if (!WARN_ON(q->base_dma & (qsz - 1))) {
 3808 		dev_info(smmu->dev, "allocated %u entries for %s\n",
 3809 			 1 << q->llq.max_n_shift, name);
 3810 	}
 3811 
 3812 	q->prod_reg	= page + prod_off;
 3813 	q->cons_reg	= page + cons_off;
 3814 	q->ent_dwords	= dwords;
 3815 
 3816 	q->q_base  = Q_BASE_RWA;
 3817 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
 3818 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
 3819 
 3820 	q->llq.prod = q->llq.cons = 0;
 3821 	return 0;
 3822 }
 3823 
 3824 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
 3825 		       struct arm_smmu_cmdq *cmdq)
 3826 {
 3827 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
 3828 
 3829 	atomic_set(&cmdq->owner_prod, 0);
 3830 	atomic_set(&cmdq->lock, 0);
 3831 
 3832 	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
 3833 							      GFP_KERNEL);
 3834 	if (!cmdq->valid_map)
 3835 		return -ENOMEM;
 3836 
 3837 	return 0;
 3838 }
 3839 
 3840 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
 3841 {
 3842 	int ret;
 3843 
 3844 	/* cmdq */
 3845 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
 3846 				      ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
 3847 				      CMDQ_ENT_DWORDS, "cmdq");
 3848 	if (ret)
 3849 		return ret;
 3850 
 3851 	ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
 3852 	if (ret)
 3853 		return ret;
 3854 
 3855 	/* evtq */
 3856 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
 3857 				      ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
 3858 				      EVTQ_ENT_DWORDS, "evtq");
 3859 	if (ret)
 3860 		return ret;
 3861 
 3862 	if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
 3863 	    (smmu->features & ARM_SMMU_FEAT_STALLS)) {
 3864 		smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
 3865 		if (!smmu->evtq.iopf)
 3866 			return -ENOMEM;
 3867 	}
 3868 
 3869 	/* priq */
 3870 	if (!(smmu->features & ARM_SMMU_FEAT_PRI))
 3871 		return 0;
 3872 
 3873 	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
 3874 				       ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
 3875 				       PRIQ_ENT_DWORDS, "priq");
 3876 }
 3877 
 3878 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
 3879 {
 3880 	u32 l1size;
 3881 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 3882 	unsigned int last_sid_idx =
 3883 		arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
 3884 
 3885 	/* Calculate the L1 size, capped to the SIDSIZE. */
 3886 	cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
 3887 	if (cfg->l2.num_l1_ents <= last_sid_idx)
 3888 		dev_warn(smmu->dev,
 3889 			 "2-level strtab only covers %u/%u bits of SID\n",
 3890 			 ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
 3891 			 smmu->sid_bits);
 3892 
 3893 	l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
 3894 	cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
 3895 					    GFP_KERNEL);
 3896 	if (!cfg->l2.l1tab) {
 3897 		dev_err(smmu->dev,
 3898 			"failed to allocate l1 stream table (%u bytes)\n",
 3899 			l1size);
 3900 		return -ENOMEM;
 3901 	}
 3902 
 3903 	cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
 3904 				      sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
 3905 	if (!cfg->l2.l2ptrs)
 3906 		return -ENOMEM;
 3907 
 3908 	return 0;
 3909 }
 3910 
 3911 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
 3912 {
 3913 	u32 size;
 3914 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 3915 
 3916 	size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
 3917 	cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
 3918 						&cfg->linear.ste_dma,
 3919 						GFP_KERNEL);
 3920 	if (!cfg->linear.table) {
 3921 		dev_err(smmu->dev,
 3922 			"failed to allocate linear stream table (%u bytes)\n",
 3923 			size);
 3924 		return -ENOMEM;
 3925 	}
 3926 	cfg->linear.num_ents = 1 << smmu->sid_bits;
 3927 
 3928 	arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
 3929 	return 0;
 3930 }
 3931 
 3932 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
 3933 {
 3934 	int ret;
 3935 
 3936 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
 3937 		ret = arm_smmu_init_strtab_2lvl(smmu);
 3938 	else
 3939 		ret = arm_smmu_init_strtab_linear(smmu);
 3940 	if (ret)
 3941 		return ret;
 3942 
 3943 	ida_init(&smmu->vmid_map);
 3944 
 3945 	return 0;
 3946 }
 3947 
 3948 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
 3949 {
 3950 	int ret;
 3951 
 3952 	mutex_init(&smmu->streams_mutex);
 3953 	smmu->streams = RB_ROOT;
 3954 
 3955 	ret = arm_smmu_init_queues(smmu);
 3956 	if (ret)
 3957 		return ret;
 3958 
 3959 	ret = arm_smmu_init_strtab(smmu);
 3960 	if (ret)
 3961 		return ret;
 3962 
 3963 	if (smmu->impl_ops && smmu->impl_ops->init_structures)
 3964 		return smmu->impl_ops->init_structures(smmu);
 3965 
 3966 	return 0;
 3967 }
 3968 
 3969 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
 3970 				   unsigned int reg_off, unsigned int ack_off)
 3971 {
 3972 	u32 reg;
 3973 
 3974 	writel_relaxed(val, smmu->base + reg_off);
 3975 	return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
 3976 					  1, ARM_SMMU_POLL_TIMEOUT_US);
 3977 }
 3978 
 3979 /* GBPA is "special" */
 3980 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
 3981 {
 3982 	int ret;
 3983 	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
 3984 
 3985 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
 3986 					 1, ARM_SMMU_POLL_TIMEOUT_US);
 3987 	if (ret)
 3988 		return ret;
 3989 
 3990 	reg &= ~clr;
 3991 	reg |= set;
 3992 	writel_relaxed(reg | GBPA_UPDATE, gbpa);
 3993 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
 3994 					 1, ARM_SMMU_POLL_TIMEOUT_US);
 3995 
 3996 	if (ret)
 3997 		dev_err(smmu->dev, "GBPA not responding to update\n");
 3998 	return ret;
 3999 }
 4000 
 4001 static void arm_smmu_free_msis(void *data)
 4002 {
 4003 	struct device *dev = data;
 4004 
 4005 	platform_device_msi_free_irqs_all(dev);
 4006 }
 4007 
 4008 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
 4009 {
 4010 	phys_addr_t doorbell;
 4011 	struct device *dev = msi_desc_to_dev(desc);
 4012 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
 4013 	phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
 4014 
 4015 	doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
 4016 	doorbell &= MSI_CFG0_ADDR_MASK;
 4017 
 4018 	writeq_relaxed(doorbell, smmu->base + cfg[0]);
 4019 	writel_relaxed(msg->data, smmu->base + cfg[1]);
 4020 	writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
 4021 }
 4022 
 4023 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
 4024 {
 4025 	int ret, nvec = ARM_SMMU_MAX_MSIS;
 4026 	struct device *dev = smmu->dev;
 4027 
 4028 	/* Clear the MSI address regs */
 4029 	writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
 4030 	writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
 4031 
 4032 	if (smmu->features & ARM_SMMU_FEAT_PRI)
 4033 		writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
 4034 	else
 4035 		nvec--;
 4036 
 4037 	if (!(smmu->features & ARM_SMMU_FEAT_MSI))
 4038 		return;
 4039 
 4040 	if (!dev->msi.domain) {
 4041 		dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
 4042 		return;
 4043 	}
 4044 
 4045 	/* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
 4046 	ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
 4047 	if (ret) {
 4048 		dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
 4049 		return;
 4050 	}
 4051 
 4052 	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
 4053 	smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
 4054 	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
 4055 
 4056 	/* Add callback to free MSIs on teardown */
 4057 	devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
 4058 }
 4059 
 4060 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
 4061 {
 4062 	int irq, ret;
 4063 
 4064 	arm_smmu_setup_msis(smmu);
 4065 
 4066 	/* Request interrupt lines */
 4067 	irq = smmu->evtq.q.irq;
 4068 	if (irq) {
 4069 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 4070 						arm_smmu_evtq_thread,
 4071 						IRQF_ONESHOT,
 4072 						"arm-smmu-v3-evtq", smmu);
 4073 		if (ret < 0)
 4074 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
 4075 	} else {
 4076 		dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
 4077 	}
 4078 
 4079 	irq = smmu->gerr_irq;
 4080 	if (irq) {
 4081 		ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
 4082 				       0, "arm-smmu-v3-gerror", smmu);
 4083 		if (ret < 0)
 4084 			dev_warn(smmu->dev, "failed to enable gerror irq\n");
 4085 	} else {
 4086 		dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
 4087 	}
 4088 
 4089 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
 4090 		irq = smmu->priq.q.irq;
 4091 		if (irq) {
 4092 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 4093 							arm_smmu_priq_thread,
 4094 							IRQF_ONESHOT,
 4095 							"arm-smmu-v3-priq",
 4096 							smmu);
 4097 			if (ret < 0)
 4098 				dev_warn(smmu->dev,
 4099 					 "failed to enable priq irq\n");
 4100 		} else {
 4101 			dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
 4102 		}
 4103 	}
 4104 }
 4105 
 4106 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
 4107 {
 4108 	int ret, irq;
 4109 	u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
 4110 
 4111 	/* Disable IRQs first */
 4112 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
 4113 				      ARM_SMMU_IRQ_CTRLACK);
 4114 	if (ret) {
 4115 		dev_err(smmu->dev, "failed to disable irqs\n");
 4116 		return ret;
 4117 	}
 4118 
 4119 	irq = smmu->combined_irq;
 4120 	if (irq) {
 4121 		/*
 4122 		 * Cavium ThunderX2 implementation doesn't support unique irq
 4123 		 * lines. Use a single irq line for all the SMMUv3 interrupts.
 4124 		 */
 4125 		ret = devm_request_threaded_irq(smmu->dev, irq,
 4126 					arm_smmu_combined_irq_handler,
 4127 					arm_smmu_combined_irq_thread,
 4128 					IRQF_ONESHOT,
 4129 					"arm-smmu-v3-combined-irq", smmu);
 4130 		if (ret < 0)
 4131 			dev_warn(smmu->dev, "failed to enable combined irq\n");
 4132 	} else
 4133 		arm_smmu_setup_unique_irqs(smmu);
 4134 
 4135 	if (smmu->features & ARM_SMMU_FEAT_PRI)
 4136 		irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
 4137 
 4138 	/* Enable interrupt generation on the SMMU */
 4139 	ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
 4140 				      ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
 4141 	if (ret)
 4142 		dev_warn(smmu->dev, "failed to enable irqs\n");
 4143 
 4144 	return 0;
 4145 }
 4146 
 4147 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
 4148 {
 4149 	int ret;
 4150 
 4151 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
 4152 	if (ret)
 4153 		dev_err(smmu->dev, "failed to clear cr0\n");
 4154 
 4155 	return ret;
 4156 }
 4157 
 4158 static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
 4159 {
 4160 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
 4161 	dma_addr_t dma;
 4162 	u32 reg;
 4163 
 4164 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
 4165 		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
 4166 				 STRTAB_BASE_CFG_FMT_2LVL) |
 4167 		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
 4168 				 ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
 4169 		      FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
 4170 		dma = cfg->l2.l1_dma;
 4171 	} else {
 4172 		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
 4173 				 STRTAB_BASE_CFG_FMT_LINEAR) |
 4174 		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
 4175 		dma = cfg->linear.ste_dma;
 4176 	}
 4177 	writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
 4178 		       smmu->base + ARM_SMMU_STRTAB_BASE);
 4179 	writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
 4180 }
 4181 
 4182 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
 4183 {
 4184 	int ret;
 4185 	u32 reg, enables;
 4186 	struct arm_smmu_cmdq_ent cmd;
 4187 
 4188 	/* Clear CR0 and sync (disables SMMU and queue processing) */
 4189 	reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
 4190 	if (reg & CR0_SMMUEN) {
 4191 		dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
 4192 		arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
 4193 	}
 4194 
 4195 	ret = arm_smmu_device_disable(smmu);
 4196 	if (ret)
 4197 		return ret;
 4198 
 4199 	/* CR1 (table and queue memory attributes) */
 4200 	reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
 4201 	      FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
 4202 	      FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
 4203 	      FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
 4204 	      FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
 4205 	      FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
 4206 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
 4207 
 4208 	/* CR2 (random crap) */
 4209 	reg = CR2_PTM | CR2_RECINVSID;
 4210 
 4211 	if (smmu->features & ARM_SMMU_FEAT_E2H)
 4212 		reg |= CR2_E2H;
 4213 
 4214 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
 4215 
 4216 	/* Stream table */
 4217 	arm_smmu_write_strtab(smmu);
 4218 
 4219 	/* Command queue */
 4220 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
 4221 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
 4222 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
 4223 
 4224 	enables = CR0_CMDQEN;
 4225 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 4226 				      ARM_SMMU_CR0ACK);
 4227 	if (ret) {
 4228 		dev_err(smmu->dev, "failed to enable command queue\n");
 4229 		return ret;
 4230 	}
 4231 
 4232 	/* Invalidate any cached configuration */
 4233 	cmd.opcode = CMDQ_OP_CFGI_ALL;
 4234 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 4235 
 4236 	/* Invalidate any stale TLB entries */
 4237 	if (smmu->features & ARM_SMMU_FEAT_HYP) {
 4238 		cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
 4239 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 4240 	}
 4241 
 4242 	cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
 4243 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 4244 
 4245 	/* Event queue */
 4246 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
 4247 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
 4248 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
 4249 
 4250 	enables |= CR0_EVTQEN;
 4251 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 4252 				      ARM_SMMU_CR0ACK);
 4253 	if (ret) {
 4254 		dev_err(smmu->dev, "failed to enable event queue\n");
 4255 		return ret;
 4256 	}
 4257 
 4258 	/* PRI queue */
 4259 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
 4260 		writeq_relaxed(smmu->priq.q.q_base,
 4261 			       smmu->base + ARM_SMMU_PRIQ_BASE);
 4262 		writel_relaxed(smmu->priq.q.llq.prod,
 4263 			       smmu->page1 + ARM_SMMU_PRIQ_PROD);
 4264 		writel_relaxed(smmu->priq.q.llq.cons,
 4265 			       smmu->page1 + ARM_SMMU_PRIQ_CONS);
 4266 
 4267 		enables |= CR0_PRIQEN;
 4268 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 4269 					      ARM_SMMU_CR0ACK);
 4270 		if (ret) {
 4271 			dev_err(smmu->dev, "failed to enable PRI queue\n");
 4272 			return ret;
 4273 		}
 4274 	}
 4275 
 4276 	if (smmu->features & ARM_SMMU_FEAT_ATS) {
 4277 		enables |= CR0_ATSCHK;
 4278 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 4279 					      ARM_SMMU_CR0ACK);
 4280 		if (ret) {
 4281 			dev_err(smmu->dev, "failed to enable ATS check\n");
 4282 			return ret;
 4283 		}
 4284 	}
 4285 
 4286 	ret = arm_smmu_setup_irqs(smmu);
 4287 	if (ret) {
 4288 		dev_err(smmu->dev, "failed to setup irqs\n");
 4289 		return ret;
 4290 	}
 4291 
 4292 	if (is_kdump_kernel())
 4293 		enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
 4294 
 4295 	/* Enable the SMMU interface */
 4296 	enables |= CR0_SMMUEN;
 4297 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 4298 				      ARM_SMMU_CR0ACK);
 4299 	if (ret) {
 4300 		dev_err(smmu->dev, "failed to enable SMMU interface\n");
 4301 		return ret;
 4302 	}
 4303 
 4304 	if (smmu->impl_ops && smmu->impl_ops->device_reset) {
 4305 		ret = smmu->impl_ops->device_reset(smmu);
 4306 		if (ret) {
 4307 			dev_err(smmu->dev, "failed to reset impl\n");
 4308 			return ret;
 4309 		}
 4310 	}
 4311 
 4312 	return 0;
 4313 }
 4314 
 4315 #define IIDR_IMPLEMENTER_ARM		0x43b
 4316 #define IIDR_PRODUCTID_ARM_MMU_600	0x483
 4317 #define IIDR_PRODUCTID_ARM_MMU_700	0x487
 4318 
 4319 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
 4320 {
 4321 	u32 reg;
 4322 	unsigned int implementer, productid, variant, revision;
 4323 
 4324 	reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
 4325 	implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
 4326 	productid = FIELD_GET(IIDR_PRODUCTID, reg);
 4327 	variant = FIELD_GET(IIDR_VARIANT, reg);
 4328 	revision = FIELD_GET(IIDR_REVISION, reg);
 4329 
 4330 	switch (implementer) {
 4331 	case IIDR_IMPLEMENTER_ARM:
 4332 		switch (productid) {
 4333 		case IIDR_PRODUCTID_ARM_MMU_600:
 4334 			/* Arm erratum 1076982 */
 4335 			if (variant == 0 && revision <= 2)
 4336 				smmu->features &= ~ARM_SMMU_FEAT_SEV;
 4337 			/* Arm erratum 1209401 */
 4338 			if (variant < 2)
 4339 				smmu->features &= ~ARM_SMMU_FEAT_NESTING;
 4340 			break;
 4341 		case IIDR_PRODUCTID_ARM_MMU_700:
 4342 			/* Arm erratum 2812531 */
 4343 			smmu->features &= ~ARM_SMMU_FEAT_BTM;
 4344 			smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
 4345 			/* Arm errata 2268618, 2812531 */
 4346 			smmu->features &= ~ARM_SMMU_FEAT_NESTING;
 4347 			break;
 4348 		}
 4349 		break;
 4350 	}
 4351 }
 4352 
 4353 static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
 4354 {
 4355 	u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
 4356 	u32 hw_features = 0;
 4357 
 4358 	switch (FIELD_GET(IDR0_HTTU, reg)) {
 4359 	case IDR0_HTTU_ACCESS_DIRTY:
 4360 		hw_features |= ARM_SMMU_FEAT_HD;
 4361 		fallthrough;
 4362 	case IDR0_HTTU_ACCESS:
 4363 		hw_features |= ARM_SMMU_FEAT_HA;
 4364 	}
 4365 
 4366 	if (smmu->dev->of_node)
 4367 		smmu->features |= hw_features;
 4368 	else if (hw_features != fw_features)
 4369 		/* ACPI IORT sets the HTTU bits */
 4370 		dev_warn(smmu->dev,
 4371 			 "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
 4372 			  hw_features, fw_features);
 4373 }
 4374 
 4375 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
 4376 {
 4377 	u32 reg;
 4378 	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
 4379 
 4380 	/* IDR0 */
 4381 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
 4382 
 4383 	/* 2-level structures */
 4384 	if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
 4385 		smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
 4386 
 4387 	if (reg & IDR0_CD2L)
 4388 		smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
 4389 
 4390 	/*
 4391 	 * Translation table endianness.
 4392 	 * We currently require the same endianness as the CPU, but this
 4393 	 * could be changed later by adding a new IO_PGTABLE_QUIRK.
 4394 	 */
 4395 	switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
 4396 	case IDR0_TTENDIAN_MIXED:
 4397 		smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
 4398 		break;
 4399 #ifdef __BIG_ENDIAN
 4400 	case IDR0_TTENDIAN_BE:
 4401 		smmu->features |= ARM_SMMU_FEAT_TT_BE;
 4402 		break;
 4403 #else
 4404 	case IDR0_TTENDIAN_LE:
 4405 		smmu->features |= ARM_SMMU_FEAT_TT_LE;
 4406 		break;
 4407 #endif
 4408 	default:
 4409 		dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
 4410 		return -ENXIO;
 4411 	}
 4412 
 4413 	/* Boolean feature flags */
 4414 	if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
 4415 		smmu->features |= ARM_SMMU_FEAT_PRI;
 4416 
 4417 	if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
 4418 		smmu->features |= ARM_SMMU_FEAT_ATS;
 4419 
 4420 	if (reg & IDR0_SEV)
 4421 		smmu->features |= ARM_SMMU_FEAT_SEV;
 4422 
 4423 	if (reg & IDR0_MSI) {
 4424 		smmu->features |= ARM_SMMU_FEAT_MSI;
 4425 		if (coherent && !disable_msipolling)
 4426 			smmu->options |= ARM_SMMU_OPT_MSIPOLL;
 4427 	}
 4428 
 4429 	if (reg & IDR0_HYP) {
 4430 		smmu->features |= ARM_SMMU_FEAT_HYP;
 4431 		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
 4432 			smmu->features |= ARM_SMMU_FEAT_E2H;
 4433 	}
 4434 
 4435 	arm_smmu_get_httu(smmu, reg);
 4436 
 4437 	/*
 4438 	 * The coherency feature as set by FW is used in preference to the ID
 4439 	 * register, but warn on mismatch.
 4440 	 */
 4441 	if (!!(reg & IDR0_COHACC) != coherent)
 4442 		dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
 4443 			 str_true_false(coherent));
 4444 
 4445 	switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
 4446 	case IDR0_STALL_MODEL_FORCE:
 4447 		smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
 4448 		fallthrough;
 4449 	case IDR0_STALL_MODEL_STALL:
 4450 		smmu->features |= ARM_SMMU_FEAT_STALLS;
 4451 	}
 4452 
 4453 	if (reg & IDR0_S1P)
 4454 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
 4455 
 4456 	if (reg & IDR0_S2P)
 4457 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
 4458 
 4459 	if (!(reg & (IDR0_S1P | IDR0_S2P))) {
 4460 		dev_err(smmu->dev, "no translation support!\n");
 4461 		return -ENXIO;
 4462 	}
 4463 
 4464 	/* We only support the AArch64 table format at present */
 4465 	switch (FIELD_GET(IDR0_TTF, reg)) {
 4466 	case IDR0_TTF_AARCH32_64:
 4467 		smmu->ias = 40;
 4468 		fallthrough;
 4469 	case IDR0_TTF_AARCH64:
 4470 		break;
 4471 	default:
 4472 		dev_err(smmu->dev, "AArch64 table format not supported!\n");
 4473 		return -ENXIO;
 4474 	}
 4475 
 4476 	/* ASID/VMID sizes */
 4477 	smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
 4478 	smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
 4479 
 4480 	/* IDR1 */
 4481 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
 4482 	if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
 4483 		dev_err(smmu->dev, "embedded implementation not supported\n");
 4484 		return -ENXIO;
 4485 	}
 4486 
 4487 	if (reg & IDR1_ATTR_TYPES_OVR)
 4488 		smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
 4489 
 4490 	/* Queue sizes, capped to ensure natural alignment */
 4491 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
 4492 					     FIELD_GET(IDR1_CMDQS, reg));
 4493 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
 4494 		/*
 4495 		 * We don't support splitting up batches, so one batch of
 4496 		 * commands plus an extra sync needs to fit inside the command
 4497 		 * queue. There's also no way we can handle the weird alignment
 4498 		 * restrictions on the base pointer for a unit-length queue.
 4499 		 */
 4500 		dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
 4501 			CMDQ_BATCH_ENTRIES);
 4502 		return -ENXIO;
 4503 	}
 4504 
 4505 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
 4506 					     FIELD_GET(IDR1_EVTQS, reg));
 4507 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
 4508 					     FIELD_GET(IDR1_PRIQS, reg));
 4509 
 4510 	/* SID/SSID sizes */
 4511 	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
 4512 	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
 4513 	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
 4514 
 4515 	/*
 4516 	 * If the SMMU supports fewer bits than would fill a single L2 stream
 4517 	 * table, use a linear table instead.
 4518 	 */
 4519 	if (smmu->sid_bits <= STRTAB_SPLIT)
 4520 		smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
 4521 
 4522 	/* IDR3 */
 4523 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
 4524 	if (FIELD_GET(IDR3_RIL, reg))
 4525 		smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
 4526 	if (FIELD_GET(IDR3_FWB, reg))
 4527 		smmu->features |= ARM_SMMU_FEAT_S2FWB;
 4528 
 4529 	if (FIELD_GET(IDR3_BBM, reg) == 2)
 4530 		smmu->features |= ARM_SMMU_FEAT_BBML2;
 4531 
 4532 	/* IDR5 */
 4533 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
 4534 
 4535 	/* Maximum number of outstanding stalls */
 4536 	smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
 4537 
 4538 	/* Page sizes */
 4539 	if (reg & IDR5_GRAN64K)
 4540 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
 4541 	if (reg & IDR5_GRAN16K)
 4542 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
 4543 	if (reg & IDR5_GRAN4K)
 4544 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
 4545 
 4546 	/* Input address size */
 4547 	if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
 4548 		smmu->features |= ARM_SMMU_FEAT_VAX;
 4549 
 4550 	/* Output address size */
 4551 	switch (FIELD_GET(IDR5_OAS, reg)) {
 4552 	case IDR5_OAS_32_BIT:
 4553 		smmu->oas = 32;
 4554 		break;
 4555 	case IDR5_OAS_36_BIT:
 4556 		smmu->oas = 36;
 4557 		break;
 4558 	case IDR5_OAS_40_BIT:
 4559 		smmu->oas = 40;
 4560 		break;
 4561 	case IDR5_OAS_42_BIT:
 4562 		smmu->oas = 42;
 4563 		break;
 4564 	case IDR5_OAS_44_BIT:
 4565 		smmu->oas = 44;
 4566 		break;
 4567 	case IDR5_OAS_52_BIT:
 4568 		smmu->oas = 52;
 4569 		smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
 4570 		break;
 4571 	default:
 4572 		dev_info(smmu->dev,
 4573 			"unknown output address size. Truncating to 48-bit\n");
 4574 		fallthrough;
 4575 	case IDR5_OAS_48_BIT:
 4576 		smmu->oas = 48;
 4577 	}
 4578 
 4579 	/* Set the DMA mask for our table walker */
 4580 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
 4581 		dev_warn(smmu->dev,
 4582 			 "failed to set DMA mask for table walker\n");
 4583 
 4584 	smmu->ias = max(smmu->ias, smmu->oas);
 4585 
 4586 	if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
 4587 	    (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 4588 		smmu->features |= ARM_SMMU_FEAT_NESTING;
 4589 
 4590 	arm_smmu_device_iidr_probe(smmu);
 4591 
 4592 	if (arm_smmu_sva_supported(smmu))
 4593 		smmu->features |= ARM_SMMU_FEAT_SVA;
 4594 
 4595 	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
 4596 		 smmu->ias, smmu->oas, smmu->features);
 4597 	return 0;
 4598 }
 4599 
 4600 #ifdef CONFIG_ACPI
 4601 #ifdef CONFIG_TEGRA241_CMDQV
 4602 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
 4603 						struct arm_smmu_device *smmu)
 4604 {
 4605 	const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
 4606 	struct acpi_device *adev;
 4607 
 4608 	/* Look for an NVDA200C node whose _UID matches the SMMU node ID */
 4609 	adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
 4610 	if (adev) {
 4611 		/* Tegra241 CMDQV driver is responsible for put_device() */
 4612 		smmu->impl_dev = &adev->dev;
 4613 		smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
 4614 		dev_info(smmu->dev, "found companion CMDQV device: %s\n",
 4615 			 dev_name(smmu->impl_dev));
 4616 	}
 4617 	kfree(uid);
 4618 }
 4619 #else
 4620 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
 4621 						struct arm_smmu_device *smmu)
 4622 {
 4623 }
 4624 #endif
 4625 
 4626 static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
 4627 				      struct arm_smmu_device *smmu)
 4628 {
 4629 	struct acpi_iort_smmu_v3 *iort_smmu =
 4630 		(struct acpi_iort_smmu_v3 *)node->node_data;
 4631 
 4632 	switch (iort_smmu->model) {
 4633 	case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
 4634 		smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
 4635 		break;
 4636 	case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
 4637 		smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
 4638 		break;
 4639 	case ACPI_IORT_SMMU_V3_GENERIC:
 4640 		/*
 4641 		 * Tegra241 implementation stores its SMMU options and impl_dev
 4642 		 * in DSDT. Thus, go through the ACPI tables unconditionally.
 4643 		 */
 4644 		acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
 4645 		break;
 4646 	}
 4647 
 4648 	dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
 4649 	return 0;
 4650 }
 4651 
 4652 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
 4653 				      struct arm_smmu_device *smmu)
 4654 {
 4655 	struct acpi_iort_smmu_v3 *iort_smmu;
 4656 	struct device *dev = smmu->dev;
 4657 	struct acpi_iort_node *node;
 4658 
 4659 	node = *(struct acpi_iort_node **)dev_get_platdata(dev);
 4660 
 4661 	/* Retrieve SMMUv3 specific data */
 4662 	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
 4663 
 4664 	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
 4665 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
 4666 
 4667 	switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
 4668 	case IDR0_HTTU_ACCESS_DIRTY:
 4669 		smmu->features |= ARM_SMMU_FEAT_HD;
 4670 		fallthrough;
 4671 	case IDR0_HTTU_ACCESS:
 4672 		smmu->features |= ARM_SMMU_FEAT_HA;
 4673 	}
 4674 
 4675 	return acpi_smmu_iort_probe_model(node, smmu);
 4676 }
 4677 #else
 4678 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
 4679 					     struct arm_smmu_device *smmu)
 4680 {
 4681 	return -ENODEV;
 4682 }
 4683 #endif
 4684 
 4685 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
 4686 				    struct arm_smmu_device *smmu)
 4687 {
 4688 	struct device *dev = &pdev->dev;
 4689 	u32 cells;
 4690 	int ret = -EINVAL;
 4691 
 4692 	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
 4693 		dev_err(dev, "missing #iommu-cells property\n");
 4694 	else if (cells != 1)
 4695 		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
 4696 	else
 4697 		ret = 0;
 4698 
 4699 	parse_driver_options(smmu);
 4700 
 4701 	if (of_dma_is_coherent(dev->of_node))
 4702 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
 4703 
 4704 	return ret;
 4705 }
 4706 
 4707 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
 4708 {
 4709 	if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
 4710 		return SZ_64K;
 4711 	else
 4712 		return SZ_128K;
 4713 }
 4714 
 4715 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
 4716 				      resource_size_t size)
 4717 {
 4718 	struct resource res = DEFINE_RES_MEM(start, size);
 4719 
 4720 	return devm_ioremap_resource(dev, &res);
 4721 }
 4722 
 4723 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
 4724 {
 4725 	struct list_head rmr_list;
 4726 	struct iommu_resv_region *e;
 4727 
 4728 	INIT_LIST_HEAD(&rmr_list);
 4729 	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
 4730 
 4731 	list_for_each_entry(e, &rmr_list, list) {
 4732 		struct iommu_iort_rmr_data *rmr;
 4733 		int ret, i;
 4734 
 4735 		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
 4736 		for (i = 0; i < rmr->num_sids; i++) {
 4737 			ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
 4738 			if (ret) {
 4739 				dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
 4740 					rmr->sids[i]);
 4741 				continue;
 4742 			}
 4743 
 4744 			/*
 4745 			 * STE table is not programmed to HW, see
 4746 			 * arm_smmu_initial_bypass_stes()
 4747 			 */
 4748 			arm_smmu_make_bypass_ste(smmu,
 4749 				arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
 4750 		}
 4751 	}
 4752 
 4753 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
 4754 }
 4755 
 4756 static void arm_smmu_impl_remove(void *data)
 4757 {
 4758 	struct arm_smmu_device *smmu = data;
 4759 
 4760 	if (smmu->impl_ops && smmu->impl_ops->device_remove)
 4761 		smmu->impl_ops->device_remove(smmu);
 4762 }
 4763 
 4764 /*
 4765  * Probe all the compiled in implementations. Each one checks to see if it
 4766  * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
 4767  * replaces the callers. Otherwise the original is returned or ERR_PTR.
 4768  */
 4769 static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
 4770 {
 4771 	struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
 4772 	const struct arm_smmu_impl_ops *ops;
 4773 	int ret;
 4774 
 4775 	if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
 4776 		new_smmu = tegra241_cmdqv_probe(smmu);
 4777 
 4778 	if (new_smmu == ERR_PTR(-ENODEV))
 4779 		return smmu;
 4780 	if (IS_ERR(new_smmu))
 4781 		return new_smmu;
 4782 
 4783 	ops = new_smmu->impl_ops;
 4784 	if (ops) {
 4785 		/* get_viommu_size and vsmmu_init ops must be paired */
 4786 		if (WARN_ON(!ops->get_viommu_size != !ops->vsmmu_init)) {
 4787 			ret = -EINVAL;
 4788 			goto err_remove;
 4789 		}
 4790 	}
 4791 
 4792 	ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
 4793 				       new_smmu);
 4794 	if (ret)
 4795 		return ERR_PTR(ret);
 4796 	return new_smmu;
 4797 
 4798 err_remove:
 4799 	arm_smmu_impl_remove(new_smmu);
 4800 	return ERR_PTR(ret);
 4801 }
 4802 
 4803 static int arm_smmu_device_probe(struct platform_device *pdev)
 4804 {
 4805 	int irq, ret;
 4806 	struct resource *res;
 4807 	resource_size_t ioaddr;
 4808 	struct arm_smmu_device *smmu;
 4809 	struct device *dev = &pdev->dev;
 4810 
 4811 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
 4812 	if (!smmu)
 4813 		return -ENOMEM;
 4814 	smmu->dev = dev;
 4815 
 4816 	if (dev->of_node) {
 4817 		ret = arm_smmu_device_dt_probe(pdev, smmu);
 4818 	} else {
 4819 		ret = arm_smmu_device_acpi_probe(pdev, smmu);
 4820 	}
 4821 	if (ret)
 4822 		return ret;
 4823 
 4824 	smmu = arm_smmu_impl_probe(smmu);
 4825 	if (IS_ERR(smmu))
 4826 		return PTR_ERR(smmu);
 4827 
 4828 	/* Base address */
 4829 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 4830 	if (!res)
 4831 		return -EINVAL;
 4832 	if (resource_size(res) < arm_smmu_resource_size(smmu)) {
 4833 		dev_err(dev, "MMIO region too small (%pr)\n", res);
 4834 		return -EINVAL;
 4835 	}
 4836 	ioaddr = res->start;
 4837 
 4838 	/*
 4839 	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
 4840 	 * the PMCG registers which are reserved by the PMU driver.
 4841 	 */
 4842 	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
 4843 	if (IS_ERR(smmu->base))
 4844 		return PTR_ERR(smmu->base);
 4845 
 4846 	if (arm_smmu_resource_size(smmu) > SZ_64K) {
 4847 		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
 4848 					       ARM_SMMU_REG_SZ);
 4849 		if (IS_ERR(smmu->page1))
 4850 			return PTR_ERR(smmu->page1);
 4851 	} else {
 4852 		smmu->page1 = smmu->base;
 4853 	}
 4854 
 4855 	/* Interrupt lines */
 4856 
 4857 	irq = platform_get_irq_byname_optional(pdev, "combined");
 4858 	if (irq > 0)
 4859 		smmu->combined_irq = irq;
 4860 	else {
 4861 		irq = platform_get_irq_byname_optional(pdev, "eventq");
 4862 		if (irq > 0)
 4863 			smmu->evtq.q.irq = irq;
 4864 
 4865 		irq = platform_get_irq_byname_optional(pdev, "priq");
 4866 		if (irq > 0)
 4867 			smmu->priq.q.irq = irq;
 4868 
 4869 		irq = platform_get_irq_byname_optional(pdev, "gerror");
 4870 		if (irq > 0)
 4871 			smmu->gerr_irq = irq;
 4872 	}
 4873 	/* Probe the h/w */
 4874 	ret = arm_smmu_device_hw_probe(smmu);
 4875 	if (ret)
 4876 		return ret;
 4877 
 4878 	/* Initialise in-memory data structures */
 4879 	ret = arm_smmu_init_structures(smmu);
 4880 	if (ret)
 4881 		goto err_free_iopf;
 4882 
 4883 	/* Record our private device structure */
 4884 	platform_set_drvdata(pdev, smmu);
 4885 
 4886 	/* Check for RMRs and install bypass STEs if any */
 4887 	arm_smmu_rmr_install_bypass_ste(smmu);
 4888 
 4889 	/* Reset the device */
 4890 	ret = arm_smmu_device_reset(smmu);
 4891 	if (ret)
 4892 		goto err_disable;
 4893 
 4894 	/* And we're up. Go go go! */
 4895 	ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
 4896 				     "smmu3.%pa", &ioaddr);
 4897 	if (ret)
 4898 		goto err_disable;
 4899 
 4900 	ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
 4901 	if (ret) {
 4902 		dev_err(dev, "Failed to register iommu\n");
 4903 		goto err_free_sysfs;
 4904 	}
 4905 
 4906 	return 0;
 4907 
 4908 err_free_sysfs:
 4909 	iommu_device_sysfs_remove(&smmu->iommu);
 4910 err_disable:
 4911 	arm_smmu_device_disable(smmu);
 4912 err_free_iopf:
 4913 	iopf_queue_free(smmu->evtq.iopf);
 4914 	return ret;
 4915 }
 4916 
 4917 static void arm_smmu_device_remove(struct platform_device *pdev)
 4918 {
 4919 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
 4920 
 4921 	iommu_device_unregister(&smmu->iommu);
 4922 	iommu_device_sysfs_remove(&smmu->iommu);
 4923 	arm_smmu_device_disable(smmu);
 4924 	iopf_queue_free(smmu->evtq.iopf);
 4925 	ida_destroy(&smmu->vmid_map);
 4926 }
 4927 
 4928 static void arm_smmu_device_shutdown(struct platform_device *pdev)
 4929 {
 4930 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
 4931 
 4932 	arm_smmu_device_disable(smmu);
 4933 }
 4934 
 4935 static const struct of_device_id arm_smmu_of_match[] = {
 4936 	{ .compatible = "arm,smmu-v3", },
 4937 	{ },
 4938 };
 4939 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
 4940 
 4941 static void arm_smmu_driver_unregister(struct platform_driver *drv)
 4942 {
 4943 	arm_smmu_sva_notifier_synchronize();
 4944 	platform_driver_unregister(drv);
 4945 }
 4946 
 4947 static struct platform_driver arm_smmu_driver = {
 4948 	.driver	= {
 4949 		.name			= "arm-smmu-v3",
 4950 		.of_match_table		= arm_smmu_of_match,
 4951 		.suppress_bind_attrs	= true,
 4952 	},
 4953 	.probe	= arm_smmu_device_probe,
 4954 	.remove = arm_smmu_device_remove,
 4955 	.shutdown = arm_smmu_device_shutdown,
 4956 };
 4957 module_driver(arm_smmu_driver, platform_driver_register,
 4958 	      arm_smmu_driver_unregister);
 4959 
 4960 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
 4961 MODULE_AUTHOR("Will Deacon <[email protected]>");
 4962 MODULE_ALIAS("platform:arm-smmu-v3");
 4963 MODULE_LICENSE("GPL v2");