From: Leon Romanovsky leonro@nvidia.com
P2PDMA assumes every client issues Untranslated Requests. A device with ATS enabled may translate any address it is handed, including a bus address, and Translation Blocking can reject its Translated Requests on a route the Untranslated answer called usable.
Unless the client declared per-mapping ATS, take the Address Type from its ATS Enable bit. Never hand such a client bus addresses: only an IOVA survives translation, and whatever it issues untranslated goes through the host bridge, whose route must therefore be usable. Apply this after the cache, which keeps holding answers that depend only on the topology.
Signed-off-by: Leon Romanovsky leonro@nvidia.com --- Documentation/driver-api/pci/p2pdma.rst | 8 ++++++ drivers/pci/p2pdma.c | 50 +++++++++++++++++++++++++++++++-- include/linux/pci-p2pdma.h | 3 +- 3 files changed, 58 insertions(+), 3 deletions(-)
diff --git a/Documentation/driver-api/pci/p2pdma.rst b/Documentation/driver-api/pci/p2pdma.rst index 7a7f29f455b0..bf9f96c07b58 100644 --- a/Documentation/driver-api/pci/p2pdma.rst +++ b/Documentation/driver-api/pci/p2pdma.rst @@ -40,6 +40,14 @@ A client therefore describes its traffic with ``enum pci_p2pdma_tlp_flags`` and asks ``pci_p2pdma_map_type_tlp()``. ``pci_p2pdma_map_type()`` answers for the default: strictly ordered Requests carrying an Untranslated address.
+The PCIe ATS Enable bit covers the whole device, and most devices translate +any address they are handed once it is set. Unless a driver has declared with +``pcim_p2pdma_set_ats_per_mapping()`` that its device chooses ATS per mapping, +that bit rather than the flags decides the Address Type, and a client with ATS +enabled is never handed a bus address, which it would translate as though it +were an IOVA. Its Translated Requests may still route directly, but anything +it issues untranslated reaches the host bridge, so that route has to work too. + The two directions are decided independently. Translation Blocking (TB), Direct Translated P2P (DT), Request Redirect (RR) and Egress Control (EC) on the client-side port decide the Request: diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index ae676509f9f3..ca79af2e64c7 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -1595,6 +1595,41 @@ void pcim_p2pdma_set_ats_per_mapping(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(pcim_p2pdma_set_ats_per_mapping);
+static unsigned int pci_p2pdma_client_tlp_flags(struct pci_dev *client, + unsigned int tlp_flags) +{ + if (client->ats_enabled) + return tlp_flags | PCI_P2PDMA_TLP_TRANSLATED; + + return tlp_flags & ~PCI_P2PDMA_TLP_TRANSLATED; +} + +/* + * A client that translates every address it is handed cannot be handed a bus + * address, which it would translate as though it were an IOVA. Its Translated + * Requests may still route directly, but a Request it issues without a + * translation carries the IOVA to the host bridge, so that route has to work + * as well. + */ +static enum pci_p2pdma_map_type +pci_p2pdma_client_map_type(struct pci_dev *provider, struct pci_dev *client, + bool per_mapping, enum pci_p2pdma_map_type type) +{ + if (per_mapping || !client->ats_enabled || + type != PCI_P2PDMA_MAP_BUS_ADDR) + return type; + + pci_dbg(client, + "P2PDMA ACS: provider=%s bus address withheld; ATS is enabled for the whole client\n", + pci_name(provider)); + + if (cpu_supports_p2pdma() || + host_bridge_whitelist(provider, client, false)) + return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; + + return PCI_P2PDMA_MAP_NOT_SUPPORTED; +} + /** * pci_p2pdma_map_type_tlp - Determine the mapping type for P2PDMA transfers * @provider: P2PDMA provider structure @@ -1609,6 +1644,11 @@ EXPORT_SYMBOL_GPL(pcim_p2pdma_set_ats_per_mapping); * ACS routes a peer-to-peer transaction by the attributes its TLPs carry, so * the answer depends on @tlp_flags. A caller that passes flags its traffic * does not match gets a mapping the fabric will not deliver. + * + * Only a client whose driver called pcim_p2pdma_set_ats_per_mapping() takes + * the Address Type from @tlp_flags. For any other client its ATS Enable bit + * decides, and a client that translates every address it is handed never gets + * %PCI_P2PDMA_MAP_BUS_ADDR. */ enum pci_p2pdma_map_type pci_p2pdma_map_type_tlp(struct p2pdma_provider *provider, struct device *dev, @@ -1620,6 +1660,7 @@ pci_p2pdma_map_type_tlp(struct p2pdma_provider *provider, struct device *dev, struct pci_p2pdma *p2pdma; struct pci_dev *client; bool provider_state; + bool per_mapping; int dist;
if (!pdev->p2pdma) { @@ -1639,6 +1680,9 @@ pci_p2pdma_map_type_tlp(struct p2pdma_provider *provider, struct device *dev, cache_index = map_types_idx(client);
rcu_read_lock(); + /* The declaration belongs to the client, the cache to the provider. */ + p2pdma = rcu_dereference(client->p2pdma); + per_mapping = p2pdma && p2pdma->ats_per_mapping; p2pdma = rcu_dereference(pdev->p2pdma);
if (p2pdma) @@ -1646,6 +1690,8 @@ pci_p2pdma_map_type_tlp(struct p2pdma_provider *provider, struct device *dev, cache_index)); provider_state = !!p2pdma; rcu_read_unlock(); + if (!per_mapping) + tlp_flags = pci_p2pdma_client_tlp_flags(client, tlp_flags); type = pci_p2pdma_map_types_unpack(cached, tlp_flags); pci_dbg(client, "P2PDMA ACS: map lookup provider=%s index=%#lx tlp-flags=%#x cached=%s(%d) provider-state=%u\n", @@ -1653,10 +1699,10 @@ pci_p2pdma_map_type_tlp(struct p2pdma_provider *provider, struct device *dev, pci_p2pdma_map_type_name(type), type, provider_state);
if (type == PCI_P2PDMA_MAP_UNKNOWN) - return calc_map_type_and_dist(pdev, client, &dist, tlp_flags, + type = calc_map_type_and_dist(pdev, client, &dist, tlp_flags, true);
- return type; + return pci_p2pdma_client_map_type(pdev, client, per_mapping, type); }
void __pci_p2pdma_update_state(struct pci_p2pdma_map_state *state, diff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h index 543a09426a11..8d96659df9df 100644 --- a/include/linux/pci-p2pdma.h +++ b/include/linux/pci-p2pdma.h @@ -207,7 +207,8 @@ static inline struct pci_dev *pci_p2pmem_find(struct device *client) * @dev: Client device that initiates the transfer * * Same as pci_p2pdma_map_type_tlp() for a client issuing strictly ordered - * Requests that carry an Untranslated address. + * Requests. Their Address Type is Untranslated unless the client enables ATS + * for the whole device. */ static inline enum pci_p2pdma_map_type pci_p2pdma_map_type(struct p2pdma_provider *provider, struct device *dev)