From: Dave Jiang dave.jiang@intel.com
commit 4f302642b70c1348773fe7e3ded9fc315fa92990 upstream.
The current implementation may miss completions after we unmask the interrupt. In order to make sure we process all competions, we need to: 1. Do an MMIO read from the device as a barrier to ensure that all PCI writes for completions have arrived. 2. Check for any additional completions that we missed.
Fixes: 8f47d1a5e545 ("dmaengine: idxd: connect idxd to dmaengine subsystem")
Reported-by: Sanjay Kumar sanjay.k.kumar@intel.com Signed-off-by: Dave Jiang dave.jiang@intel.com Link: https://lore.kernel.org/r/158834641769.35613.1341160109892008587.stgit@djian... Signed-off-by: Vinod Koul vkoul@kernel.org Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org
--- drivers/dma/idxd/device.c | 7 +++++++ drivers/dma/idxd/irq.c | 26 +++++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-)
--- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -62,6 +62,13 @@ int idxd_unmask_msix_vector(struct idxd_ perm.ignore = 0; iowrite32(perm.bits, idxd->reg_base + offset);
+ /* + * A readback from the device ensures that any previously generated + * completion record writes are visible to software based on PCI + * ordering rules. + */ + perm.bits = ioread32(idxd->reg_base + offset); + return 0; }
--- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -173,6 +173,7 @@ static int irq_process_pending_llist(str struct llist_node *head; int queued = 0;
+ *processed = 0; head = llist_del_all(&irq_entry->pending_llist); if (!head) return 0; @@ -197,6 +198,7 @@ static int irq_process_work_list(struct struct list_head *node, *next; int queued = 0;
+ *processed = 0; if (list_empty(&irq_entry->work_list)) return 0;
@@ -218,10 +220,9 @@ static int irq_process_work_list(struct return queued; }
-irqreturn_t idxd_wq_thread(int irq, void *data) +static int idxd_desc_process(struct idxd_irq_entry *irq_entry) { - struct idxd_irq_entry *irq_entry = data; - int rc, processed = 0, retry = 0; + int rc, processed, total = 0;
/* * There are two lists we are processing. The pending_llist is where @@ -244,15 +245,26 @@ irqreturn_t idxd_wq_thread(int irq, void */ do { rc = irq_process_work_list(irq_entry, &processed); - if (rc != 0) { - retry++; + total += processed; + if (rc != 0) continue; - }
rc = irq_process_pending_llist(irq_entry, &processed); - } while (rc != 0 && retry != 10); + total += processed; + } while (rc != 0); + + return total; +} + +irqreturn_t idxd_wq_thread(int irq, void *data) +{ + struct idxd_irq_entry *irq_entry = data; + int processed;
+ processed = idxd_desc_process(irq_entry); idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); + /* catch anything unprocessed after unmasking */ + processed += idxd_desc_process(irq_entry);
if (processed == 0) return IRQ_NONE;