|
Hannes Reinecke |
c70c1a |
From: Keith Busch <kbusch@kernel.org>
|
|
Hannes Reinecke |
c70c1a |
Date: Mon, 2 Mar 2020 08:56:53 -0800
|
|
Hannes Reinecke |
c70c1a |
Subject: [PATCH] nvme-pci: Remove two-pass completions
|
|
Hannes Reinecke |
c70c1a |
Git-commit: 324b494c286298d51bc5ed5107644ebe23f9dad6
|
|
Hannes Reinecke |
c70c1a |
Patch-mainline: v5.7-rc1
|
|
Hannes Reinecke |
c70c1a |
References: git-fixes
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
Completion handling had been done in two steps: find all new completions
|
|
Hannes Reinecke |
c70c1a |
under a lock, then handle those completions outside the lock. This was
|
|
Hannes Reinecke |
c70c1a |
done to make the locked section as short as possible so that other
|
|
Hannes Reinecke |
c70c1a |
threads using the same lock wait less time.
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
The driver no longer shares locks during completion, and is in fact
|
|
Hannes Reinecke |
c70c1a |
lockless for interrupt driven queues, so the optimization no longer
|
|
Hannes Reinecke |
c70c1a |
serves its original purpose. Replace the two-pass completion queue
|
|
Hannes Reinecke |
c70c1a |
handler with a single pass that completes entries immediately.
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
|
|
Hannes Reinecke |
c70c1a |
Reviewed-by: Christoph Hellwig <hch@lst.de>
|
|
Hannes Reinecke |
c70c1a |
Signed-off-by: Keith Busch <kbusch@kernel.org>
|
|
Hannes Reinecke |
c70c1a |
Acked-by: Hannes Reinecke <hare@suse.com>
|
|
Hannes Reinecke |
c70c1a |
---
|
|
Hannes Reinecke |
c70c1a |
drivers/nvme/host/pci.c | 42 ++++++++++-------------------------------
|
|
Hannes Reinecke |
c70c1a |
1 file changed, 10 insertions(+), 32 deletions(-)
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
|
|
Hannes Reinecke |
c70c1a |
index 98d8ddd7aa0f..02f22c63adcf 100644
|
|
Hannes Reinecke |
c70c1a |
--- a/drivers/nvme/host/pci.c
|
|
Hannes Reinecke |
c70c1a |
+++ b/drivers/nvme/host/pci.c
|
|
Hannes Reinecke |
c70c1a |
@@ -971,15 +971,6 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
|
|
Hannes Reinecke |
c70c1a |
nvme_end_request(req, cqe->status, cqe->result);
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
-static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
|
|
Hannes Reinecke |
c70c1a |
-{
|
|
Hannes Reinecke |
c70c1a |
- while (start != end) {
|
|
Hannes Reinecke |
c70c1a |
- nvme_handle_cqe(nvmeq, start);
|
|
Hannes Reinecke |
c70c1a |
- if (++start == nvmeq->q_depth)
|
|
Hannes Reinecke |
c70c1a |
- start = 0;
|
|
Hannes Reinecke |
c70c1a |
- }
|
|
Hannes Reinecke |
c70c1a |
-}
|
|
Hannes Reinecke |
c70c1a |
-
|
|
Hannes Reinecke |
c70c1a |
static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
if (++nvmeq->cq_head == nvmeq->q_depth) {
|
|
Hannes Reinecke |
c70c1a |
@@ -988,19 +979,17 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
-static inline int nvme_process_cq(struct nvme_queue *nvmeq, u16 *start,
|
|
Hannes Reinecke |
c70c1a |
- u16 *end)
|
|
Hannes Reinecke |
c70c1a |
+static inline int nvme_process_cq(struct nvme_queue *nvmeq)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
int found = 0;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
- *start = nvmeq->cq_head;
|
|
Hannes Reinecke |
c70c1a |
while (nvme_cqe_pending(nvmeq)) {
|
|
Hannes Reinecke |
c70c1a |
found++;
|
|
Hannes Reinecke |
c70c1a |
+ nvme_handle_cqe(nvmeq, nvmeq->cq_head);
|
|
Hannes Reinecke |
c70c1a |
nvme_update_cq_head(nvmeq);
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
- *end = nvmeq->cq_head;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
- if (*start != *end)
|
|
Hannes Reinecke |
c70c1a |
+ if (found)
|
|
Hannes Reinecke |
c70c1a |
nvme_ring_cq_doorbell(nvmeq);
|
|
Hannes Reinecke |
c70c1a |
return found;
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
@@ -1009,21 +998,16 @@ static irqreturn_t nvme_irq(int irq, void *data)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
struct nvme_queue *nvmeq = data;
|
|
Hannes Reinecke |
c70c1a |
irqreturn_t ret = IRQ_NONE;
|
|
Hannes Reinecke |
c70c1a |
- u16 start, end;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
/*
|
|
Hannes Reinecke |
c70c1a |
* The rmb/wmb pair ensures we see all updates from a previous run of
|
|
Hannes Reinecke |
c70c1a |
* the irq handler, even if that was on another CPU.
|
|
Hannes Reinecke |
c70c1a |
*/
|
|
Hannes Reinecke |
c70c1a |
rmb();
|
|
Hannes Reinecke |
c70c1a |
- nvme_process_cq(nvmeq, &start, &end;;
|
|
Hannes Reinecke |
c70c1a |
+ if (nvme_process_cq(nvmeq))
|
|
Hannes Reinecke |
c70c1a |
+ ret = IRQ_HANDLED;
|
|
Hannes Reinecke |
c70c1a |
wmb();
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
- if (start != end) {
|
|
Hannes Reinecke |
c70c1a |
- nvme_complete_cqes(nvmeq, start, end);
|
|
Hannes Reinecke |
c70c1a |
- return IRQ_HANDLED;
|
|
Hannes Reinecke |
c70c1a |
- }
|
|
Hannes Reinecke |
c70c1a |
-
|
|
Hannes Reinecke |
c70c1a |
return ret;
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
@@ -1042,7 +1026,6 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
|
|
Hannes Reinecke |
c70c1a |
static int nvme_poll_irqdisable(struct nvme_queue *nvmeq)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
|
|
Hannes Reinecke |
c70c1a |
- u16 start, end;
|
|
Hannes Reinecke |
c70c1a |
int found;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
/*
|
|
Hannes Reinecke |
c70c1a |
@@ -1052,29 +1035,27 @@ static int nvme_poll_irqdisable(struct nvme_queue *nvmeq)
|
|
Hannes Reinecke |
c70c1a |
*/
|
|
Hannes Reinecke |
c70c1a |
if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) {
|
|
Hannes Reinecke |
c70c1a |
spin_lock(&nvmeq->cq_poll_lock);
|
|
Hannes Reinecke |
c70c1a |
- found = nvme_process_cq(nvmeq, &start, &end;;
|
|
Hannes Reinecke |
c70c1a |
+ found = nvme_process_cq(nvmeq);
|
|
Hannes Reinecke |
c70c1a |
spin_unlock(&nvmeq->cq_poll_lock);
|
|
Hannes Reinecke |
c70c1a |
} else {
|
|
Hannes Reinecke |
c70c1a |
disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
|
|
Hannes Reinecke |
c70c1a |
- found = nvme_process_cq(nvmeq, &start, &end;;
|
|
Hannes Reinecke |
c70c1a |
+ found = nvme_process_cq(nvmeq);
|
|
Hannes Reinecke |
c70c1a |
enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
- nvme_complete_cqes(nvmeq, start, end);
|
|
Hannes Reinecke |
c70c1a |
return found;
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
static int nvme_poll(struct blk_mq_hw_ctx *hctx)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
struct nvme_queue *nvmeq = hctx->driver_data;
|
|
Hannes Reinecke |
c70c1a |
- u16 start, end;
|
|
Hannes Reinecke |
c70c1a |
bool found;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
if (!nvme_cqe_pending(nvmeq))
|
|
Hannes Reinecke |
c70c1a |
return 0;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
spin_lock(&nvmeq->cq_poll_lock);
|
|
Hannes Reinecke |
c70c1a |
- found = nvme_process_cq(nvmeq, &start, &end;;
|
|
Hannes Reinecke |
c70c1a |
+ found = nvme_process_cq(nvmeq);
|
|
Hannes Reinecke |
c70c1a |
spin_unlock(&nvmeq->cq_poll_lock);
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
return found;
|
|
Hannes Reinecke |
c70c1a |
@@ -1406,13 +1387,10 @@ static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
|
|
Hannes Reinecke |
c70c1a |
*/
|
|
Hannes Reinecke |
c70c1a |
static void nvme_reap_pending_cqes(struct nvme_dev *dev)
|
|
Hannes Reinecke |
c70c1a |
{
|
|
Hannes Reinecke |
c70c1a |
- u16 start, end;
|
|
Hannes Reinecke |
c70c1a |
int i;
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
- for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
|
|
Hannes Reinecke |
c70c1a |
- nvme_process_cq(&dev->queues[i], &start, &end;;
|
|
Hannes Reinecke |
c70c1a |
- nvme_complete_cqes(&dev->queues[i], start, end);
|
|
Hannes Reinecke |
c70c1a |
- }
|
|
Hannes Reinecke |
c70c1a |
+ for (i = dev->ctrl.queue_count - 1; i > 0; i--)
|
|
Hannes Reinecke |
c70c1a |
+ nvme_process_cq(&dev->queues[i]);
|
|
Hannes Reinecke |
c70c1a |
}
|
|
Hannes Reinecke |
c70c1a |
|
|
Hannes Reinecke |
c70c1a |
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
|
|
Hannes Reinecke |
c70c1a |
--
|
|
Hannes Reinecke |
c70c1a |
2.29.2
|
|
Hannes Reinecke |
c70c1a |
|