Add an arm_smmu_viommu_cache_invalidate() function for user space to issue cache invalidation commands via viommu.
The viommu invalidation takes the same native format of a 128-bit command, as the hwpt invalidation. Thus, reuse the same driver data structure, but make it wider to accept CMDQ_OP_ATC_INV and CMDQ_OP_CFGI_CD{_ALL}.
Scan the commands against the supported ist and fix the VMIDs and SIDs.
Signed-off-by: Nicolin Chen nicolinc@nvidia.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 76 ++++++++++++++++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + include/uapi/linux/iommufd.h | 7 +- 3 files changed, 80 insertions(+), 4 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a2af693bc7b2..bddbb98da414 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3267,15 +3267,32 @@ static void arm_smmu_domain_nested_free(struct iommu_domain *domain) kfree(container_of(domain, struct arm_smmu_nested_domain, domain)); }
+static int arm_smmu_convert_viommu_vdev_id(struct iommufd_viommu *viommu, + u32 vdev_id, u32 *sid) +{ + struct arm_smmu_master *master; + struct device *dev; + + dev = iommufd_viommu_find_device(viommu, vdev_id); + if (!dev) + return -EIO; + master = dev_iommu_priv_get(dev); + + if (sid) + *sid = master->streams[0].id; + return 0; +} + /* * Convert, in place, the raw invalidation command into an internal format that * can be passed to arm_smmu_cmdq_issue_cmdlist(). Internally commands are * stored in CPU endian. * - * Enforce the VMID on the command. + * Enforce the VMID or the SID on the command. */ static int arm_smmu_convert_user_cmd(struct arm_smmu_domain *s2_parent, + struct iommufd_viommu *viommu, struct iommu_hwpt_arm_smmuv3_invalidate *cmd) { u16 vmid = s2_parent->s2_cfg.vmid; @@ -3297,13 +3314,46 @@ arm_smmu_convert_user_cmd(struct arm_smmu_domain *s2_parent, cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID; cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vmid); break; + case CMDQ_OP_ATC_INV: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_CD_ALL: + if (viommu) { + u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd[0]); + + if (arm_smmu_convert_viommu_vdev_id(viommu, vsid, &sid)) + return -EIO; + cmd->cmd[0] &= ~CMDQ_CFGI_0_SID; + cmd->cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + break; + } + fallthrough; default: return -EIO; } return 0; }
+static inline bool +arm_smmu_must_lock_vdev_id(struct iommu_hwpt_arm_smmuv3_invalidate *cmds, + unsigned int num_cmds) +{ + int i; + + for (i = 0; i < num_cmds; i++) { + switch (cmds[i].cmd[0] & CMDQ_0_OP) { + case CMDQ_OP_ATC_INV: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_CD_ALL: + return true; + default: + continue; + } + } + return false; +} + static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent, + struct iommufd_viommu *viommu, struct iommu_user_data_array *array) { struct arm_smmu_device *smmu = s2_parent->smmu; @@ -3313,6 +3363,7 @@ static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent, struct iommu_hwpt_arm_smmuv3_invalidate *end; struct arm_smmu_cmdq_ent ent; struct arm_smmu_cmdq *cmdq; + bool must_lock = false; int ret;
/* A zero-length array is allowed to validate the array type */ @@ -3335,12 +3386,17 @@ static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent, if (ret) goto out;
+ if (viommu) + must_lock = arm_smmu_must_lock_vdev_id(cmds, array->entry_num); + if (must_lock) + iommufd_viommu_lock_vdev_id(viommu); + ent.opcode = cmds->cmd[0] & CMDQ_0_OP; cmdq = arm_smmu_get_cmdq(smmu, &ent);
last_batch = cmds; while (cur != end) { - ret = arm_smmu_convert_user_cmd(s2_parent, cur); + ret = arm_smmu_convert_user_cmd(s2_parent, viommu, cur); if (ret) goto out;
@@ -3358,6 +3414,8 @@ static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent, last_batch = cur; } out: + if (must_lock) + iommufd_viommu_unlock_vdev_id(viommu); array->entry_num = cur - cmds; kfree(cmds); return ret; @@ -3370,7 +3428,7 @@ static int arm_smmu_cache_invalidate_user(struct iommu_domain *domain, container_of(domain, struct arm_smmu_nested_domain, domain);
return __arm_smmu_cache_invalidate_user( - nested_domain->s2_parent, array); + nested_domain->s2_parent, NULL, array); }
static const struct iommu_domain_ops arm_smmu_nested_ops = { @@ -3863,6 +3921,15 @@ static int arm_smmu_def_domain_type(struct device *dev) return 0; }
+static int arm_smmu_viommu_cache_invalidate(struct iommufd_viommu *viommu, + struct iommu_user_data_array *array) +{ + struct iommu_domain *domain = iommufd_viommu_to_parent_domain(viommu); + + return __arm_smmu_cache_invalidate_user( + to_smmu_domain(domain), viommu, array); +} + static struct iommu_ops arm_smmu_ops = { .identity_domain = &arm_smmu_identity_domain, .blocked_domain = &arm_smmu_blocked_domain, @@ -3893,6 +3960,9 @@ static struct iommu_ops arm_smmu_ops = { .iotlb_sync = arm_smmu_iotlb_sync, .iova_to_phys = arm_smmu_iova_to_phys, .free = arm_smmu_domain_free_paging, + .default_viommu_ops = &(const struct iommufd_viommu_ops) { + .cache_invalidate = arm_smmu_viommu_cache_invalidate, + } } };
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 6c8ae70c90fe..e7f6e9194a9e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -10,6 +10,7 @@
#include <linux/bitfield.h> #include <linux/iommu.h> +#include <linux/iommufd.h> #include <linux/kernel.h> #include <linux/mmzone.h> #include <linux/sizes.h> diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index f3aefb11f681..0d973486b604 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -734,13 +734,18 @@ struct iommu_hwpt_vtd_s1_invalidate { * @cmd: 128-bit cache invalidation command that runs in SMMU CMDQ. * Must be little-endian. * - * Supported command list: + * Supported command list when passing in a HWPT via @hwpt_id: * CMDQ_OP_TLBI_NSNH_ALL * CMDQ_OP_TLBI_NH_VA * CMDQ_OP_TLBI_NH_VAA * CMDQ_OP_TLBI_NH_ALL * CMDQ_OP_TLBI_NH_ASID * + * Additional to the list above, when passing in a VIOMMU via @hwpt_id: + * CMDQ_OP_ATC_INV + * CMDQ_OP_CFGI_CD + * CMDQ_OP_CFGI_CD_ALL + * * -EIO will be returned if the command is not supported. */ struct iommu_hwpt_arm_smmuv3_invalidate {