From: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>

Date: Fri, 4 Aug 2017 13:54:35 -0700

Subject: IB/rdmavt, hfi1, qib: Enhance rdmavt and hfi1 to use 32 bit lids

Patch-mainline: v4.14-rc1

Git-commit: 51e658f5dd362cc8666f3f5ec1986660e3e51047

References: bsc#1060463 FATE#323043 bsc#1027067 FATE#322649 bsc#1058717 FATE#322789

Increase lid used in hfi1 driver to 32 bits. qib continues

to use 16 bit lids.

Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>

Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>

Signed-off-by: Don Hiatt <don.hiatt@intel.com>

Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>

Signed-off-by: Doug Ledford <dledford@redhat.com>

Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>

---

 drivers/infiniband/hw/hfi1/chip.c   |   12 +++-

 drivers/infiniband/hw/hfi1/hfi.h    |    2 

 drivers/infiniband/hw/hfi1/mad.c    |   91 +++++++++++++++++++++++++++++++-----

 drivers/infiniband/hw/hfi1/verbs.c  |   23 ---------

 drivers/infiniband/hw/hfi1/verbs.h  |    2 

 drivers/infiniband/hw/qib/qib_mad.c |    4 -

 include/rdma/rdma_vt.h              |    2 

 7 files changed, 93 insertions(+), 43 deletions(-)

--- a/drivers/infiniband/hw/hfi1/chip.c

+++ b/drivers/infiniband/hw/hfi1/chip.c

@@ -10067,10 +10067,16 @@ static void set_lidlmc(struct hfi1_pport

 	struct hfi1_devdata *dd = ppd->dd;

 	u32 mask = ~((1U << ppd->lmc) - 1);

 	u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);

+	u32 lid;

+	/*

+	 * Program 0 in CSR if port lid is extended. This prevents

+	 * 9B packets being sent out for large lids.

+	 */

+	lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;

 	c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK

 		| DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);

-	c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)

+	c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)

 			<< DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |

 	      ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)

 			<< DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);

@@ -10081,7 +10087,7 @@ static void set_lidlmc(struct hfi1_pport

 	 */

 	sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<

 			SEND_CTXT_CHECK_SLID_MASK_SHIFT) |

-	       (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<

+	       (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<

 			SEND_CTXT_CHECK_SLID_VALUE_SHIFT);

 	for (i = 0; i < dd->chip_send_contexts; i++) {

@@ -10091,7 +10097,7 @@ static void set_lidlmc(struct hfi1_pport

 	}

 	/* Now we have to do the same thing for the sdma engines */

-	sdma_update_lmc(dd, mask, ppd->lid);

+	sdma_update_lmc(dd, mask, lid);

 }

 static const char *state_completed_string(u32 completed)

--- a/drivers/infiniband/hw/hfi1/hfi.h

+++ b/drivers/infiniband/hw/hfi1/hfi.h

@@ -718,7 +718,7 @@ struct hfi1_pportdata {

 	u32 ibmaxlen;

 	u32 current_egress_rate; /* units [10^6 bits/sec] */

 	/* LID programmed for this instance */

-	u16 lid;

+	u32 lid;

 	/* list of pkeys programmed; 0 if not set */

 	u16 pkeys[MAX_PKEY_VALUES];

 	u16 link_width_supported;

--- a/drivers/infiniband/hw/hfi1/mad.c

+++ b/drivers/infiniband/hw/hfi1/mad.c

@@ -234,6 +234,61 @@ static void subn_handle_opa_trap_repress

 	spin_unlock_irqrestore(&ibp->rvp.lock, flags);

 }

+static void hfi1_update_sm_ah_attr(struct hfi1_ibport *ibp,

+				   struct rdma_ah_attr *attr, u32 dlid)

+{

+	rdma_ah_set_dlid(attr, dlid);

+	rdma_ah_set_port_num(attr, ppd_from_ibp(ibp)->port);

+	if (dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {

+		struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

+

+		rdma_ah_set_ah_flags(attr, IB_AH_GRH);

+		grh->sgid_index = 0;

+		grh->hop_limit = 1;

+		grh->dgid.global.subnet_prefix =

+			ibp->rvp.gid_prefix;

+		grh->dgid.global.interface_id = OPA_MAKE_ID(dlid);

+	}

+}

+

+static int hfi1_modify_qp0_ah(struct hfi1_ibport *ibp,

+			      struct rvt_ah *ah, u32 dlid)

+{

+	struct rdma_ah_attr attr;

+	struct rvt_qp *qp0;

+	int ret = -EINVAL;

+

+	memset(&attr, 0, sizeof(attr));

+	attr.type = ah->ibah.type;

+	hfi1_update_sm_ah_attr(ibp, &attr, dlid);

+	rcu_read_lock();

+	qp0 = rcu_dereference(ibp->rvp.qp[0]);

+	if (qp0)

+		ret = rdma_modify_ah(&ah->ibah, &attr);

+	rcu_read_unlock();

+	return ret;

+}

+

+static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid)

+{

+	struct rdma_ah_attr attr;

+	struct ib_ah *ah = ERR_PTR(-EINVAL);

+	struct rvt_qp *qp0;

+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);

+	struct hfi1_devdata *dd = dd_from_ppd(ppd);

+	u8 port_num = ppd->port;

+

+	memset(&attr, 0, sizeof(attr));

+	attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);

+	hfi1_update_sm_ah_attr(ibp, &attr, dlid);

+	rcu_read_lock();

+	qp0 = rcu_dereference(ibp->rvp.qp[0]);

+	if (qp0)

+		ah = rdma_create_ah(qp0->ibqp.pd, &attr);

+	rcu_read_unlock();

+	return ah;

+}

+

 static void send_trap(struct hfi1_ibport *ibp, struct trap_node *trap)

 {

 	struct ib_mad_send_buf *send_buf;

@@ -1283,8 +1338,8 @@ static int __subn_set_opa_portinfo(struc

 	struct hfi1_ibport *ibp;

 	u8 clientrereg;

 	unsigned long flags;

-	u32 smlid, opa_lid; /* tmp vars to hold LID values */

-	u16 lid;

+	u32 smlid;

+	u32 lid;

 	u8 ls_old, ls_new, ps_new;

 	u8 vls;

 	u8 msl;

@@ -1301,22 +1356,20 @@ static int __subn_set_opa_portinfo(struc

 		return reply((struct ib_mad_hdr *)smp);

 	}

-	opa_lid = be32_to_cpu(pi->lid);

-	if (opa_lid & 0xFFFF0000) {

-		pr_warn("OPA_PortInfo lid out of range: %X\n", opa_lid);

+	lid = be32_to_cpu(pi->lid);

+	if (lid & 0xFF000000) {

+		pr_warn("OPA_PortInfo lid out of range: %X\n", lid);

 		smp->status |= IB_SMP_INVALID_FIELD;

 		goto get_only;

 	}

-	lid = (u16)(opa_lid & 0x0000FFFF);

 	smlid = be32_to_cpu(pi->sm_lid);

-	if (smlid & 0xFFFF0000) {

+	if (smlid & 0xFF000000) {

 		pr_warn("OPA_PortInfo SM lid out of range: %X\n", smlid);

 		smp->status |= IB_SMP_INVALID_FIELD;

 		goto get_only;

 	}

-	smlid &= 0x0000FFFF;

 	clientrereg = (pi->clientrereg_subnettimeout &

 			OPA_PI_MASK_CLIENT_REREGISTER);

@@ -1331,12 +1384,16 @@ static int __subn_set_opa_portinfo(struc

 	ls_old = driver_lstate(ppd);

 	ibp->rvp.mkey = pi->mkey;

-	ibp->rvp.gid_prefix = pi->subnet_prefix;

+	if (ibp->rvp.gid_prefix != pi->subnet_prefix) {

+		ibp->rvp.gid_prefix = pi->subnet_prefix;

+		event.event = IB_EVENT_GID_CHANGE;

+		ib_dispatch_event(&event);

+	}

 	ibp->rvp.mkey_lease_period = be16_to_cpu(pi->mkey_lease_period);

 	/* Must be a valid unicast LID address. */

 	if ((lid == 0 && ls_old > IB_PORT_INIT) ||

-	    lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {

+	     (hfi1_is_16B_mcast(lid))) {

 		smp->status |= IB_SMP_INVALID_FIELD;

 		pr_warn("SubnSet(OPA_PortInfo) lid invalid 0x%x\n",

 			lid);

@@ -1349,6 +1406,16 @@ static int __subn_set_opa_portinfo(struc

 		hfi1_set_lid(ppd, lid, pi->mkeyprotect_lmc & OPA_PI_MASK_LMC);

 		event.event = IB_EVENT_LID_CHANGE;

 		ib_dispatch_event(&event);

+

+		if (HFI1_PORT_GUID_INDEX + 1 < HFI1_GUIDS_PER_PORT) {

+			/* Manufacture GID from LID to support extended

+			 * addresses

+			 */

+			ppd->guids[HFI1_PORT_GUID_INDEX + 1] =

+				be64_to_cpu(OPA_MAKE_ID(lid));

+			event.event = IB_EVENT_GID_CHANGE;

+			ib_dispatch_event(&event);

+		}

 	}

 	msl = pi->smsl & OPA_PI_MASK_SMSL;

@@ -1359,7 +1426,7 @@ static int __subn_set_opa_portinfo(struc

 	/* Must be a valid unicast LID address. */

 	if ((smlid == 0 && ls_old > IB_PORT_INIT) ||

-	    smlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {

+	     (hfi1_is_16B_mcast(smlid))) {

 		smp->status |= IB_SMP_INVALID_FIELD;

 		pr_warn("SubnSet(OPA_PortInfo) smlid invalid 0x%x\n", smlid);

 	} else if (smlid != ibp->rvp.sm_lid || msl != ibp->rvp.sm_sl) {

@@ -1367,7 +1434,7 @@ static int __subn_set_opa_portinfo(struc

 		spin_lock_irqsave(&ibp->rvp.lock, flags);

 		if (ibp->rvp.sm_ah) {

 			if (smlid != ibp->rvp.sm_lid)

-				rdma_ah_set_dlid(&ibp->rvp.sm_ah->attr, smlid);

+				hfi1_modify_qp0_ah(ibp, ibp->rvp.sm_ah, smlid);

 			if (msl != ibp->rvp.sm_sl)

 				rdma_ah_set_sl(&ibp->rvp.sm_ah->attr, msl);

 		}

--- a/drivers/infiniband/hw/hfi1/verbs.c

+++ b/drivers/infiniband/hw/hfi1/verbs.c

@@ -1394,7 +1394,7 @@ static int query_port(struct rvt_dev_inf

 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);

 	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);

 	struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];

-	u16 lid = ppd->lid;

+	u32 lid = ppd->lid;

 	/* props being zeroed by the caller, avoid zeroing it here */

 	props->lid = lid ? lid : 0;

@@ -1555,27 +1555,6 @@ static void hfi1_notify_new_ah(struct ib

 		ah->log_pmtu = ilog2(dd->vld[ah->vl].mtu);

 }

-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid)

-{

-	struct rdma_ah_attr attr;

-	struct ib_ah *ah = ERR_PTR(-EINVAL);

-	struct rvt_qp *qp0;

-	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);

-	struct hfi1_devdata *dd = dd_from_ppd(ppd);

-	u8 port_num = ppd->port;

-

-	memset(&attr, 0, sizeof(attr));

-	attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);

-	rdma_ah_set_dlid(&attr, dlid);

-	rdma_ah_set_port_num(&attr, ppd_from_ibp(ibp)->port);

-	rcu_read_lock();

-	qp0 = rcu_dereference(ibp->rvp.qp[0]);

-	if (qp0)

-		ah = rdma_create_ah(qp0->ibqp.pd, &attr);

-	rcu_read_unlock();

-	return ah;

-}

-

 /**

  * hfi1_get_npkeys - return the size of the PKEY table for context 0

  * @dd: the hfi1_ib device

--- a/drivers/infiniband/hw/hfi1/verbs.h

+++ b/drivers/infiniband/hw/hfi1/verbs.h

@@ -334,8 +334,6 @@ void hfi1_rc_hdrerr(

 u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);

-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid);

-

 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);

 void hfi1_ud_rcv(struct hfi1_packet *packet);

--- a/drivers/infiniband/hw/qib/qib_mad.c

+++ b/drivers/infiniband/hw/qib/qib_mad.c

@@ -105,7 +105,7 @@ static void qib_send_trap(struct qib_ibp

 		if (ibp->rvp.sm_lid != be16_to_cpu(IB_LID_PERMISSIVE)) {

 			struct ib_ah *ah;

-			ah = qib_create_qp0_ah(ibp, ibp->rvp.sm_lid);

+			ah = qib_create_qp0_ah(ibp, (u16)ibp->rvp.sm_lid);

 			if (IS_ERR(ah))

 				ret = PTR_ERR(ah);

 			else {

@@ -496,7 +496,7 @@ static int subn_get_portinfo(struct ib_s

 		pip->mkey = ibp->rvp.mkey;

 	pip->gid_prefix = ibp->rvp.gid_prefix;

 	pip->lid = cpu_to_be16(ppd->lid);

-	pip->sm_lid = cpu_to_be16(ibp->rvp.sm_lid);

+	pip->sm_lid = cpu_to_be16((u16)ibp->rvp.sm_lid);

 	pip->cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);

 	/* pip->diag_code; */

 	pip->mkey_lease_period = cpu_to_be16(ibp->rvp.mkey_lease_period);

--- a/include/rdma/rdma_vt.h

+++ b/include/rdma/rdma_vt.h

@@ -91,7 +91,7 @@ struct rvt_ibport {

 	__be16 pma_counter_select[5];

 	u16 pma_tag;

 	u16 mkey_lease_period;

-	u16 sm_lid;

+	u32 sm_lid;

 	u8 sm_sl;

 	u8 mkeyprot;

 	u8 subnet_timeout;