From: James Smart <jsmart2021@gmail.com>

Date: Fri, 14 May 2021 12:55:54 -0700

Subject: scsi: lpfc: Fix node handling for Fabric Controller and Domain

 Controller

Patch-mainline: Queued in subsystem maintainer repository

Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git

Git-commit: fe83e3b9b422ac8ece2359c7b7290efe7f0335a2

References: bsc#1186451

During link bounce testing, RPI counts were seen to differ from the number

of nodes. For fabric and domain controllers, a temporary RPI is assigned,

but the code isn't registering it. If the nodes do go away, such as on link

down, the temporary RPI isn't being released.

Change the way these two fabric services are managed, make them behave like

any other remote port. Register the RPI and register with the transport.

Never leave the nodes in a NPR or UNUSED state where their RPI is in limbo.

This allows them to follow normal dev_loss_tmo handling, RPI refcounting,

and normal removal rules. It also allows fabric I/Os to use the RPI for

traffic requests.

Note: There is some logic that still has a couple of exceptions when the

Domain controller (0xfffcXX). There are cases where the fabric won't have a

valid login but will send RDP. Other times, it will it send a LOGO then an

RDP. It makes for ad-hoc behavior to manage the node. Exceptions are

documented in the code.

Link: https://lore.kernel.org/r/20210514195559.119853-7-jsmart2021@gmail.com

Co-developed-by: Justin Tee <justin.tee@broadcom.com>

Signed-off-by: Justin Tee <justin.tee@broadcom.com>

Signed-off-by: James Smart <jsmart2021@gmail.com>

Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

Acked-by: Daniel Wagner <dwagner@suse.de>

---

 drivers/scsi/lpfc/lpfc_crtn.h      |    1 

 drivers/scsi/lpfc/lpfc_debugfs.c   |    4 -

 drivers/scsi/lpfc/lpfc_disc.h      |    1 

 drivers/scsi/lpfc/lpfc_els.c       |  137 +++++++++++++++++++++++++++++++++----

 drivers/scsi/lpfc/lpfc_hbadisc.c   |   54 +++++++++++++-

 drivers/scsi/lpfc/lpfc_nportdisc.c |   11 ++

 drivers/scsi/lpfc/lpfc_sli.c       |    1 

 7 files changed, 189 insertions(+), 20 deletions(-)

--- a/drivers/scsi/lpfc/lpfc_crtn.h

+++ b/drivers/scsi/lpfc/lpfc_crtn.h

@@ -80,6 +80,7 @@ void lpfc_mbx_cmpl_reg_login(struct lpfc

 void lpfc_mbx_cmpl_dflt_rpi(struct lpfc_hba *, LPFC_MBOXQ_t *);

 void lpfc_mbx_cmpl_fabric_reg_login(struct lpfc_hba *, LPFC_MBOXQ_t *);

 void lpfc_mbx_cmpl_ns_reg_login(struct lpfc_hba *, LPFC_MBOXQ_t *);

+void lpfc_mbx_cmpl_fc_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb);

 void lpfc_mbx_cmpl_fdmi_reg_login(struct lpfc_hba *, LPFC_MBOXQ_t *);

 void lpfc_mbx_cmpl_reg_vfi(struct lpfc_hba *, LPFC_MBOXQ_t *);

 void lpfc_unregister_vfi_cmpl(struct lpfc_hba *, LPFC_MBOXQ_t *);

--- a/drivers/scsi/lpfc/lpfc_debugfs.c

+++ b/drivers/scsi/lpfc/lpfc_debugfs.c

@@ -863,10 +863,10 @@ lpfc_debugfs_nodelist_data(struct lpfc_v

 		len += scnprintf(buf+len, size-len, "%s DID:x%06x ",

 				statep, ndlp->nlp_DID);

 		len += scnprintf(buf+len, size-len,

-				"WWPN x%llx ",

+				"WWPN x%016llx ",

 				wwn_to_u64(ndlp->nlp_portname.u.wwn));

 		len += scnprintf(buf+len, size-len,

-				"WWNN x%llx ",

+				"WWNN x%016llx ",

 				wwn_to_u64(ndlp->nlp_nodename.u.wwn));

 		len += scnprintf(buf+len, size-len, "RPI:x%04x ",

 				 ndlp->nlp_rpi);

--- a/drivers/scsi/lpfc/lpfc_disc.h

+++ b/drivers/scsi/lpfc/lpfc_disc.h

@@ -86,6 +86,7 @@ enum lpfc_fc4_xpt_flags {

 struct lpfc_nodelist {

 	struct list_head nlp_listp;

+	struct serv_parm fc_sparam;		/* buffer for service params */

 	struct lpfc_name nlp_portname;

 	struct lpfc_name nlp_nodename;

--- a/drivers/scsi/lpfc/lpfc_els.c

+++ b/drivers/scsi/lpfc/lpfc_els.c

@@ -3086,6 +3086,95 @@ lpfc_cmpl_els_cmd(struct lpfc_hba *phba,

 }

 /**

+ * lpfc_reg_fab_ctrl_node - RPI register the fabric controller node.

+ * @vport: pointer to lpfc_vport data structure.

+ * @fc_ndlp: pointer to the fabric controller (0xfffffd) node.

+ *

+ * This routine registers the rpi assigned to the fabric controller

+ * NPort_ID (0xfffffd) with the port and moves the node to UNMAPPED

+ * state triggering a registration with the SCSI transport.

+ *

+ * This routine is single out because the fabric controller node

+ * does not receive a PLOGI.  This routine is consumed by the

+ * SCR and RDF ELS commands.  Callers are expected to qualify

+ * with SLI4 first.

+ **/

+static int

+lpfc_reg_fab_ctrl_node(struct lpfc_vport *vport, struct lpfc_nodelist *fc_ndlp)

+{

+	int rc = 0;

+	struct lpfc_hba *phba = vport->phba;

+	struct lpfc_nodelist *ns_ndlp;

+	LPFC_MBOXQ_t *mbox;

+	struct lpfc_dmabuf *mp;

+

+	if (fc_ndlp->nlp_flag & NLP_RPI_REGISTERED)

+		return rc;

+

+	ns_ndlp = lpfc_findnode_did(vport, NameServer_DID);

+	if (!ns_ndlp)

+		return -ENODEV;

+

+	lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,

+			 "0935 %s: Reg FC RPI x%x on FC DID x%x NSSte: x%x\n",

+			 __func__, fc_ndlp->nlp_rpi, fc_ndlp->nlp_DID,

+			 ns_ndlp->nlp_state);

+	if (ns_ndlp->nlp_state != NLP_STE_UNMAPPED_NODE)

+		return -ENODEV;

+

+	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);

+	if (!mbox) {

+		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,

+				 "0936 %s: no memory for reg_login "

+				 "Data: x%x x%x x%x x%x\n", __func__,

+				 fc_ndlp->nlp_DID, fc_ndlp->nlp_state,

+				 fc_ndlp->nlp_flag, fc_ndlp->nlp_rpi);

+		return -ENOMEM;

+	}

+	rc = lpfc_reg_rpi(phba, vport->vpi, fc_ndlp->nlp_DID,

+			  (u8 *)&vport->fc_sparam, mbox, fc_ndlp->nlp_rpi);

+	if (rc) {

+		rc = -EACCES;

+		goto out;

+	}

+

+	fc_ndlp->nlp_flag |= NLP_REG_LOGIN_SEND;

+	mbox->mbox_cmpl = lpfc_mbx_cmpl_fc_reg_login;

+	mbox->ctx_ndlp = lpfc_nlp_get(fc_ndlp);

+	if (!mbox->ctx_ndlp) {

+		rc = -ENOMEM;

+		goto out_mem;

+	}

+

+	mbox->vport = vport;

+	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);

+	if (rc == MBX_NOT_FINISHED) {

+		rc = -ENODEV;

+		lpfc_nlp_put(fc_ndlp);

+		goto out_mem;

+	}

+	/* Success path. Exit. */

+	lpfc_nlp_set_state(vport, fc_ndlp,

+			   NLP_STE_REG_LOGIN_ISSUE);

+	return 0;

+

+ out_mem:

+	fc_ndlp->nlp_flag &= ~NLP_REG_LOGIN_SEND;

+	mp = (struct lpfc_dmabuf *)mbox->ctx_buf;

+	lpfc_mbuf_free(phba, mp->virt, mp->phys);

+	kfree(mp);

+

+ out:

+	mempool_free(mbox, phba->mbox_mem_pool);

+	lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,

+			 "0938 %s: failed to format reg_login "

+			 "Data: x%x x%x x%x x%x\n", __func__,

+			 fc_ndlp->nlp_DID, fc_ndlp->nlp_state,

+			 fc_ndlp->nlp_flag, fc_ndlp->nlp_rpi);

+	return rc;

+}

+

+/**

  * lpfc_cmpl_els_disc_cmd - Completion callback function for Discovery ELS cmd

  * @phba: pointer to lpfc hba data structure.

  * @cmdiocb: pointer to lpfc command iocb data structure.

@@ -3231,10 +3320,18 @@ lpfc_issue_els_scr(struct lpfc_vport *vp

 	elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp,

 				     ndlp->nlp_DID, ELS_CMD_SCR);

-

 	if (!elsiocb)

 		return 1;

+	if (phba->sli_rev == LPFC_SLI_REV4) {

+		rc = lpfc_reg_fab_ctrl_node(vport, ndlp);

+		if (rc) {

+			lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,

+					 "0937 %s: Failed to reg fc node, rc %d\n",

+					 __func__, rc);

+			return 1;

+		}

+	}

 	pcmd = (uint8_t *) (((struct lpfc_dmabuf *) elsiocb->context2)->virt);

 	*((uint32_t *) (pcmd)) = ELS_CMD_SCR;

@@ -3522,6 +3619,17 @@ lpfc_issue_els_rdf(struct lpfc_vport *vp

 	if (!elsiocb)

 		return -ENOMEM;

+	if (phba->sli_rev == LPFC_SLI_REV4 &&

+	    !(ndlp->nlp_flag & NLP_RPI_REGISTERED)) {

+		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE,

+				 "0939 %s: FC_NODE x%x RPI x%x flag x%x "

+				 "ste x%x type x%x Not registered\n",

+				 __func__, ndlp->nlp_DID, ndlp->nlp_rpi,

+				 ndlp->nlp_flag, ndlp->nlp_state,

+				 ndlp->nlp_type);

+		return -ENODEV;

+	}

+

 	/* Configure the payload for the supported FPIN events. */

 	prdf = (struct lpfc_els_rdf_req *)

 		(((struct lpfc_dmabuf *)elsiocb->context2)->virt);

@@ -4396,7 +4504,6 @@ lpfc_cmpl_els_logo_acc(struct lpfc_hba *

 	struct lpfc_nodelist *ndlp = (struct lpfc_nodelist *) cmdiocb->context1;

 	struct lpfc_vport *vport = cmdiocb->vport;

 	IOCB_t *irsp;

-	u32 xpt_flags = 0, did_mask = 0;

 	irsp = &rspiocb->iocb;

 	lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_RSP,

@@ -4409,6 +4516,15 @@ lpfc_cmpl_els_logo_acc(struct lpfc_hba *

 			 ndlp->nlp_DID, kref_read(&ndlp->kref), ndlp->nlp_flag,

 			 ndlp->nlp_state, ndlp->nlp_rpi);

+	/* This clause allows the LOGO ACC to complete and free resources

+	 * for the Fabric Domain Controller.  It does deliberately skip

+	 * the unreg_rpi and release rpi because some fabrics send RDP

+	 * requests after logging out from the initiator.

+	 */

+	if (ndlp->nlp_type & NLP_FABRIC &&

+	    ((ndlp->nlp_DID & WELL_KNOWN_DID_MASK) != WELL_KNOWN_DID_MASK))

+		goto out;

+

 	if (ndlp->nlp_state == NLP_STE_NPR_NODE) {

 		/* NPort Recovery mode or node is just allocated */

 		if (!lpfc_nlp_not_used(ndlp)) {

@@ -4416,16 +4532,11 @@ lpfc_cmpl_els_logo_acc(struct lpfc_hba *

 			 * If this a fabric node that cleared its transport

 			 * registration, release the rpi.

 			 */

-			xpt_flags = SCSI_XPT_REGD | NVME_XPT_REGD;

-			did_mask = ndlp->nlp_DID & Fabric_DID_MASK;

-			if (did_mask == Fabric_DID_MASK &&

-			    !(ndlp->fc4_xpt_flags & xpt_flags)) {

-				spin_lock_irq(&ndlp->lock);

-				ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;

-				if (phba->sli_rev == LPFC_SLI_REV4)

-					ndlp->nlp_flag |= NLP_RELEASE_RPI;

-				spin_unlock_irq(&ndlp->lock);

-			}

+			spin_lock_irq(&ndlp->lock);

+			ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;

+			if (phba->sli_rev == LPFC_SLI_REV4)

+				ndlp->nlp_flag |= NLP_RELEASE_RPI;

+			spin_unlock_irq(&ndlp->lock);

 			lpfc_unreg_rpi(vport, ndlp);

 		} else {

 			/* Indicate the node has already released, should

@@ -4434,7 +4545,7 @@ lpfc_cmpl_els_logo_acc(struct lpfc_hba *

 			cmdiocb->context1 = NULL;

 		}

 	}

-

+ out:

 	/*

 	 * The driver received a LOGO from the rport and has ACK'd it.

 	 * At this point, the driver is done so release the IOCB

--- a/drivers/scsi/lpfc/lpfc_hbadisc.c

+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c

@@ -77,9 +77,7 @@ static int

 lpfc_valid_xpt_node(struct lpfc_nodelist *ndlp)

 {

 	if (ndlp->nlp_fc4_type ||

-	    ndlp->nlp_DID == Fabric_DID ||

-	    ndlp->nlp_DID == NameServer_DID ||

-	    ndlp->nlp_DID == FDMI_DID)

+	    ndlp->nlp_type & NLP_FABRIC)

 		return 1;

 	return 0;

 }

@@ -826,7 +824,8 @@ lpfc_cleanup_rpis(struct lpfc_vport *vpo

 		if ((phba->sli3_options & LPFC_SLI3_VPORT_TEARDOWN) ||

 		    ((vport->port_type == LPFC_NPIV_PORT) &&

 		     ((ndlp->nlp_DID == NameServer_DID) ||

-		      (ndlp->nlp_DID == FDMI_DID))))

+		      (ndlp->nlp_DID == FDMI_DID) ||

+		      (ndlp->nlp_DID == Fabric_Cntl_DID))))

 			lpfc_unreg_rpi(vport, ndlp);

 		/* Leave Fabric nodes alone on link down */

@@ -4161,6 +4160,53 @@ lpfc_mbx_cmpl_ns_reg_login(struct lpfc_h

 	return;

 }

+/*

+ * This routine handles processing a Fabric Controller REG_LOGIN mailbox

+ * command upon completion. It is setup in the LPFC_MBOXQ

+ * as the completion routine when the command is handed off to the SLI layer.

+ */

+void

+lpfc_mbx_cmpl_fc_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)

+{

+	struct lpfc_vport *vport = pmb->vport;

+	MAILBOX_t *mb = &pmb->u.mb;

+	struct lpfc_dmabuf *mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);

+	struct lpfc_nodelist *ndlp;

+

+	ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;

+	pmb->ctx_ndlp = NULL;

+	pmb->ctx_buf = NULL;

+

+	if (mb->mbxStatus) {

+		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,

+				 "0933 %s: Register FC login error: 0x%x\n",

+				 __func__, mb->mbxStatus);

+		goto out;

+	}

+

+	if (phba->sli_rev < LPFC_SLI_REV4)

+		ndlp->nlp_rpi = mb->un.varWords[0];

+

+	lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,

+			 "0934 %s: Complete FC x%x RegLogin rpi x%x ste x%x\n",

+			 __func__, ndlp->nlp_DID, ndlp->nlp_rpi,

+			 ndlp->nlp_state);

+

+	ndlp->nlp_flag |= NLP_RPI_REGISTERED;

+	ndlp->nlp_type |= NLP_FABRIC;

+	lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);

+

+ out:

+	lpfc_mbuf_free(phba, mp->virt, mp->phys);

+	kfree(mp);

+	mempool_free(pmb, phba->mbox_mem_pool);

+

+	/* Drop the reference count from the mbox at the end after

+	 * all the current reference to the ndlp have been done.

+	 */

+	lpfc_nlp_put(ndlp);

+}

+

 static void

 lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)

 {

--- a/drivers/scsi/lpfc/lpfc_nportdisc.c

+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c

@@ -785,6 +785,15 @@ lpfc_rcv_logo(struct lpfc_vport *vport,

 	else

 		lpfc_els_rsp_acc(vport, ELS_CMD_ACC, cmdiocb, ndlp, NULL);

+	/* This clause allows the initiator to ACC the LOGO back to the

+	 * Fabric Domain Controller.  It does deliberately skip all other

+	 * steps because some fabrics send RDP requests after logging out

+	 * from the initiator.

+	 */

+	if (ndlp->nlp_type & NLP_FABRIC &&

+	    ((ndlp->nlp_DID & WELL_KNOWN_DID_MASK) != WELL_KNOWN_DID_MASK))

+		return 0;

+

 	/* Notify transport of connectivity loss to trigger cleanup. */

 	if (phba->nvmet_support &&

 	    ndlp->nlp_state == NLP_STE_UNMAPPED_NODE)

@@ -1423,6 +1432,8 @@ lpfc_cmpl_plogi_plogi_issue(struct lpfc_

 		switch (ndlp->nlp_DID) {

 		case NameServer_DID:

 			mbox->mbox_cmpl = lpfc_mbx_cmpl_ns_reg_login;

+			/* Fabric Controller Node needs these parameters. */

+			memcpy(&ndlp->fc_sparam, sp, sizeof(struct serv_parm));

 			break;

 		case FDMI_DID:

 			mbox->mbox_cmpl = lpfc_mbx_cmpl_fdmi_reg_login;

--- a/drivers/scsi/lpfc/lpfc_sli.c

+++ b/drivers/scsi/lpfc/lpfc_sli.c

@@ -2755,7 +2755,6 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc

 				} else {

 					__lpfc_sli_rpi_release(vport, ndlp);

 				}

-

 				lpfc_nlp_put(ndlp);

 			}

 		}