From: Leon Romanovsky leonro@nvidia.com
ACS Translation Blocking rejects upstream Memory Requests with a Translated address before peer-routing controls apply. Ignoring it can report routes that reject the Requester's address type. Intel SPT PCH Root Ports store ACS Control at a nonstandard dword offset, so a standard word read also misses Translation Blocking.
Scan every port traversed by each route and reject blocked Translated classes without host-bridge fallback. Read the Intel control dword through a device-specific accessor, preserving rejection of unreadable controls. Direct routes remain usable when only ports above the divergence block Translated Requests.
Signed-off-by: Leon Romanovsky leonro@nvidia.com --- drivers/pci/p2pdma.c | 117 ++++++++++++++++++++++++++++++++++++++++++++++----- drivers/pci/pci.h | 6 +++ drivers/pci/quirks.c | 15 +++++++ 3 files changed, 128 insertions(+), 10 deletions(-)
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 2d5a9488da41b..0203ef86fa86a 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -559,11 +559,12 @@ enum pci_acs_p2pdma_state { PCI_ACS_P2PDMA_NOT_SUPPORTED, PCI_ACS_P2PDMA_DIRECT, PCI_ACS_P2PDMA_REDIRECT, + PCI_ACS_P2PDMA_BLOCKED, };
/* * Decide how a peer-to-peer Request at an ACS-capable ingress port routes, - * from that port's ACS Control register. + * from that port's ACS Control register and the Request's Address Type. * * Linux does not read the Egress Control Vector, so Egress Control is treated * conservatively as a redirect. Per PCIe r7.0 Table 6-11 the outcomes it @@ -573,6 +574,18 @@ enum pci_acs_p2pdma_state { static enum pci_acs_p2pdma_state pci_acs_p2pdma_request(u16 ctrl, unsigned int tlp_flags) { + if (tlp_flags & PCI_P2PDMA_TLP_TRANSLATED) { + /* + * PCIe r7.0 sec 6.12.1.1: Translation Blocking makes every + * Upstream Memory Request whose Address Type is not + * Untranslated an ACS Violation, taking precedence over the + * P2P controls. Sec 7.7.12.5: Direct Translated P2P "is + * ignored if ACS Translation Blocking Enable is 1b". + */ + if (ctrl & PCI_ACS_TB) + return PCI_ACS_P2PDMA_BLOCKED; + } + return ctrl & (PCI_ACS_RR | PCI_ACS_EC) ? PCI_ACS_P2PDMA_REDIRECT : PCI_ACS_P2PDMA_DIRECT; } @@ -603,7 +616,7 @@ pci_acs_p2pdma_completion(u16 ctrl, unsigned int tlp_flags) */ static bool pci_acs_p2pdma_ctrl(struct pci_dev *pdev, u16 *ctrl) { - int pos; + int pos, ret;
pos = pdev->acs_cap; if (!pos) { @@ -611,8 +624,10 @@ static bool pci_acs_p2pdma_ctrl(struct pci_dev *pdev, u16 *ctrl) return true; }
- if (pci_read_config_word(pdev, pos + PCI_ACS_CTRL, ctrl) || - PCI_POSSIBLE_ERROR(*ctrl)) + ret = pci_dev_specific_read_acs_ctrl(pdev, ctrl); + if (ret == -ENOTTY) + ret = pci_read_config_word(pdev, pos + PCI_ACS_CTRL, ctrl); + if (ret || PCI_POSSIBLE_ERROR(*ctrl)) return false;
*ctrl = pci_acs_p2pdma_quirk_redirect(pdev, *ctrl); @@ -681,6 +696,39 @@ pci_acs_p2pdma_no_upstream_forwarding(struct pci_dev *common, return NULL; }
+/* + * Report whether any port between @client and @common rejects Translated + * addresses. If @common is NULL, every port up to the host bridge is walked. + * A port whose ACS Control cannot be read counts as blocking, which withdraws + * only the Translated classes because an Untranslated Request is routed by the + * redirect controls instead. + */ +static bool pci_p2pdma_path_blocks_translation(struct pci_dev *client, + struct pci_dev *common) +{ + struct pci_dev *pdev; + u16 ctrl; + + /* + * @common is @client itself when the provider is the client or sits + * below it. The Request never travels upstream then, so no port sees + * it and none can reject its Address Type. + */ + if (client == common) + return false; + + for (pdev = pci_upstream_bridge(client); pdev && pdev != common; + pdev = pci_upstream_bridge(pdev)) { + if (!pci_acs_p2pdma_ctrl(pdev, &ctrl)) + return true; + + if (ctrl & PCI_ACS_TB) + return true; + } + + return false; +} + static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev) { if (!buf) @@ -747,6 +795,13 @@ static void pci_p2pdma_warn_redir(struct pci_dev *client, * @cpl_quirk: A device quirk reports Completion Redirect at that port. * @req_upstream: The Requester has no internal route to its sibling. * @cpl_upstream: The Completer has no internal route to its sibling. + * @tb_on_path: A port below the divergence blocks Translated addresses. + * Every route out of @client passes those, so none carries them. + * @tb_above_divergence: A port at or above the divergence blocks Translated + * addresses. Only a Request continuing to the host bridge passes + * those, so a direct route is still open to them. + * @no_common_bridge: The two paths share no upstream bridge, so no direct + * route exists for ACS to gate. * @unreadable: First port whose ACS Control could not be read, if any. * @uf_blocked: First shared port without usable Upstream Forwarding, if any. * @uf_unreadable: Shared port whose forwarding control could not be read. @@ -757,15 +812,29 @@ struct pci_p2pdma_acs_path { bool cpl_quirk; bool req_upstream; bool cpl_upstream; + bool tb_on_path; + bool tb_above_divergence; + bool no_common_bridge; struct pci_dev *unreadable; struct pci_dev *uf_blocked; struct pci_dev *uf_unreadable; };
+/* + * ACS Translation Blocking is not a routing control, so unlike the redirect + * controls it is not decided at the divergence alone. PCIe r7.0 sec 6.12.1.1 + * has every Downstream Port check the Address Type of each Upstream Memory + * Request it receives, ahead of "any applicable ACS P2P control mechanisms". + * A port below the divergence cannot redirect the Request anywhere it was not + * already going, but it can still reject a Translated address. + */ static enum pci_acs_p2pdma_state pci_p2pdma_request_state(const struct pci_p2pdma_acs_path *path, unsigned int tlp_flags) { + if (tlp_flags & PCI_P2PDMA_TLP_TRANSLATED && path->tb_on_path) + return PCI_ACS_P2PDMA_BLOCKED; + if (path->req_upstream) return PCI_ACS_P2PDMA_REDIRECT;
@@ -796,11 +865,23 @@ static enum pci_p2pdma_map_type pci_p2pdma_route(const struct pci_p2pdma_acs_path *path, unsigned int tlp_flags) { + enum pci_acs_p2pdma_state req; + if (path->unreadable) return PCI_P2PDMA_MAP_NOT_SUPPORTED;
- if (pci_p2pdma_request_state(path, tlp_flags) == - PCI_ACS_P2PDMA_DIRECT && + req = pci_p2pdma_request_state(path, tlp_flags); + + /* + * Translation Blocking rejects the Address Type rather than the + * target, so a blocked Request stays blocked however it is addressed. + * No host bridge fallback keeps a Translated address working; the + * caller has to issue a different kind of Request instead. + */ + if (req == PCI_ACS_P2PDMA_BLOCKED) + return PCI_P2PDMA_MAP_NOT_SUPPORTED; + + if (!path->no_common_bridge && req == PCI_ACS_P2PDMA_DIRECT && pci_p2pdma_completion_state(path, tlp_flags) == PCI_ACS_P2PDMA_DIRECT) return PCI_P2PDMA_MAP_BUS_ADDR; @@ -808,6 +889,14 @@ pci_p2pdma_route(const struct pci_p2pdma_acs_path *path, if (path->uf_blocked || path->uf_unreadable) return PCI_P2PDMA_MAP_NOT_SUPPORTED;
+ /* + * A Request that turns around at the divergence never reaches the + * ports above it, but one that keeps climbing to the host bridge + * does, so that route has to clear their Translation Blocking too. + */ + if (tlp_flags & PCI_P2PDMA_TLP_TRANSLATED && path->tb_above_divergence) + return PCI_P2PDMA_MAP_NOT_SUPPORTED; + return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; }
@@ -1075,7 +1164,8 @@ pci_p2pdma_map_types_unpack(unsigned long val, unsigned int tlp_flags) * The client is the Requester and the provider the Completer. At the path * divergence, check Request Redirect and Egress Control on the Requester-side * port, and Completion Redirect for read Completions on the Completer-side - * port. + * port. Translation Blocking is checked on every port the Request passes + * instead, because it rejects a Request rather than routing it. * * Those controls apply to different TLPs, so every class named by &enum * pci_p2pdma_tlp_flags is decided from the one walk and cached together; @@ -1083,8 +1173,8 @@ pci_p2pdma_map_types_unpack(unsigned long val, unsigned int tlp_flags) * * If ACS redirects traffic at either divergence port, return * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE. If the ACS Control register cannot be - * read, return PCI_P2PDMA_MAP_NOT_SUPPORTED. Otherwise, return - * PCI_P2PDMA_MAP_BUS_ADDR. + * read, or Translation Blocking rejects the class being asked about, return + * PCI_P2PDMA_MAP_NOT_SUPPORTED. Otherwise, return PCI_P2PDMA_MAP_BUS_ADDR. * * Any two devices that have a data path that goes through the host bridge * will consult a whitelist. If the host bridge is in the whitelist, return @@ -1137,8 +1227,10 @@ calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client, * request can only get to the peer through the host bridge. */ *dist = dist_a + dist_b; + path.no_common_bridge = true; + path.tb_on_path = pci_p2pdma_path_blocks_translation(client, NULL); for (flags = 0; flags < PCI_P2PDMA_TLP_CLASSES; flags++) - map_type[flags] = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; + map_type[flags] = pci_p2pdma_route(&path, flags); goto map_through_host_bridge;
check_paths_acs: @@ -1169,6 +1261,11 @@ calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client, } }
+ path.tb_on_path = pci_p2pdma_path_blocks_translation(client, a); + if (b_child) + path.tb_above_divergence = + pci_p2pdma_path_blocks_translation(b_child, NULL); + /* Shared ports need forwarding only for classes using the host bridge. */ for (flags = 0; flags < PCI_P2PDMA_TLP_CLASSES; flags++) { if (pci_p2pdma_route(&path, flags) != diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 77efa4c5c960e..f800308d198f6 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1100,12 +1100,18 @@ resource_size_t pci_min_window_alignment(struct pci_bus *bus, void pci_acs_init(struct pci_dev *dev); void pci_enable_acs(struct pci_dev *dev); #ifdef CONFIG_PCI_QUIRKS +int pci_dev_specific_read_acs_ctrl(struct pci_dev *dev, u16 *ctrl); int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags); int pci_dev_specific_enable_acs(struct pci_dev *dev); int pci_dev_specific_disable_acs_redir(struct pci_dev *dev); void pci_disable_broken_acs_cap(struct pci_dev *pdev); int pcie_failed_link_retrain(struct pci_dev *dev); #else +static inline int pci_dev_specific_read_acs_ctrl(struct pci_dev *dev, u16 *ctrl) +{ + return -ENOTTY; +} + static inline int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags) { diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 05605c3afe08f..4b6e868722280 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5132,6 +5132,21 @@ static bool pci_quirk_intel_spt_pch_acs_match(struct pci_dev *dev)
#define INTEL_SPT_ACS_CTRL (PCI_ACS_CAP + 4)
+int pci_dev_specific_read_acs_ctrl(struct pci_dev *dev, u16 *ctrl) +{ + u32 val = ~0; + int ret; + + if (dev->vendor != PCI_VENDOR_ID_INTEL || !dev->acs_cap || + !pci_quirk_intel_spt_pch_acs_match(dev)) + return -ENOTTY; + + ret = pci_read_config_dword(dev, dev->acs_cap + INTEL_SPT_ACS_CTRL, + &val); + *ctrl = val; + return ret; +} + static int pci_quirk_intel_spt_pch_acs(struct pci_dev *dev, u16 acs_flags) { int pos;