From: James Smart <jsmart2021@gmail.com>
Date: Thu, 24 May 2018 21:08:58 -0700
Subject: scsi: lpfc: correct oversubscription of nvme io requests for an
adapter
Git-commit: 4d5e789a2eb111d7f9e032d0ebaecb465a2eca8f
Patch-mainline: v4.18-rc1
References: bsc#1088866
Under large configurations, the driver would start to log message 6065 -
NVME out of buffers (exchanges).
The driver is using the ndlp cmd_qdepth value when determining the max
outstanding ios for an adapter. This value, by default, is set to 65536,
which exceeds the maximum exchange counts supported on an adapter. The ndlp
cmd_qdepth has no relevance and outstanding io count should be capped at
the max exchange count with IO requests beyond that level getting bounced
back with an EBUSY status so that they are retried by the block layer.
Signed-off-by: Dick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: James Smart <james.smart@broadcom.com>
Reviewed-by: Hannes Reinecke <hare@suse.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
Acked-by: Johannes Thumshirn <jthumshirn@suse.de>
---
drivers/scsi/lpfc/lpfc_attr.c | 7 +++++++
drivers/scsi/lpfc/lpfc_nportdisc.c | 14 +++++++++++++-
drivers/scsi/lpfc/lpfc_nvme.c | 23 +++++++++++++++++++----
3 files changed, 39 insertions(+), 5 deletions(-)
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -297,6 +297,13 @@ lpfc_nvme_info_show(struct device *dev,
len = snprintf(buf, PAGE_SIZE, "NVME Initiator Enabled\n");
spin_lock_irq(shost->host_lock);
+ len += snprintf(buf + len, PAGE_SIZE - len,
+ "XRI Dist lpfc%d Total %d NVME %d SCSI %d ELS %d\n",
+ phba->brd_no,
+ phba->sli4_hba.max_cfg_param.max_xri,
+ phba->sli4_hba.nvme_xri_max,
+ phba->sli4_hba.scsi_xri_max,
+ lpfc_sli4_get_els_iocb_cnt(phba));
/* Port state is only one of two values for now. */
if (localport->port_id)
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1981,8 +1981,20 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vp
ndlp->nlp_type |= NLP_NVME_TARGET;
if (bf_get_be32(prli_disc, nvpr))
ndlp->nlp_type |= NLP_NVME_DISCOVERY;
+
+ /* This node is an NVME target. Adjust the command
+ * queue depth on this node to not exceed the available
+ * xris.
+ */
+ ndlp->cmd_qdepth = phba->sli4_hba.nvme_xri_max;
+
+ /*
+ * If prli_fba is set, the Target supports FirstBurst.
+ * If prli_fb_sz is 0, the FirstBurst size is unlimited,
+ * otherwise it defines the actual size supported by
+ * the NVME Target.
+ */
if ((bf_get_be32(prli_fba, nvpr) == 1) &&
- (bf_get_be32(prli_fb_sz, nvpr) > 0) &&
(phba->cfg_nvme_enable_fb) &&
(!phba->nvmet_support)) {
/* Both sides support FB. The target's first
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -973,9 +973,22 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hb
/* Sanity check on return of outstanding command */
if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) {
+ if (!lpfc_ncmd) {
+ lpfc_printf_vlog(vport, KERN_ERR,
+ LOG_NODE | LOG_NVME_IOERR,
+ "6071 Null lpfc_ncmd pointer. No "
+ "release, skip completion\n");
+ return;
+ }
+
lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
- "6071 Completion pointers bad on wqe %p.\n",
- wcqe);
+ "6066 Missing cmpl ptrs: lpfc_ncmd %p, "
+ "nvmeCmd %p nrport %p\n",
+ lpfc_ncmd, lpfc_ncmd->nvmeCmd,
+ lpfc_ncmd->nrport);
+
+ /* Release the lpfc_ncmd regardless of the missing elements. */
+ lpfc_release_nvme_buf(phba, lpfc_ncmd);
return;
}
nCmd = lpfc_ncmd->nvmeCmd;
@@ -1537,8 +1550,10 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_l
!expedite) {
lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
"6174 Fail IO, ndlp qdepth exceeded: "
- "idx %d DID %x\n",
- lpfc_queue_info->index, ndlp->nlp_DID);
+ "idx %d DID %x pend %d qdepth %d\n",
+ lpfc_queue_info->index, ndlp->nlp_DID,
+ atomic_read(&ndlp->cmd_pending),
+ ndlp->cmd_qdepth);
atomic_inc(&lport->xmt_fcp_qdepth);
ret = -EBUSY;
goto out_fail;