From: Jason Gunthorpe <jgg@mellanox.com>
Date: Sun, 24 Jun 2018 16:57:50 +0300
Subject: IB/mlx4: Create slave AH's directly
Patch-mainline: v4.19-rc1
Git-commit: 5e62d5ff1b9a6573d0d6b411893b1c17ccc21ce7
References: bsc#1103988 FATE#326003

Since slave GID's do not exist in the core gid table we can no longer use
the core code to help do this without creating inconsistencies. Directly
create the AH using mlx4 internal APIs.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Reviewed-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
 drivers/infiniband/hw/mlx4/ah.c      |   58 ++++++++++++++++++++++++++++++-----
 drivers/infiniband/hw/mlx4/mad.c     |   22 ++-----------
 drivers/infiniband/hw/mlx4/mlx4_ib.h |    4 ++
 3 files changed, 59 insertions(+), 25 deletions(-)

--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -96,19 +96,29 @@ static struct ib_ah *create_iboe_ah(stru
 
 	memcpy(ah->av.eth.mac, ah_attr->roce.dmac, ETH_ALEN);
 	eth_zero_addr(ah->av.eth.s_mac);
+
+	/*
+	 * If sgid_attr is NULL we are being called by mlx4_ib_create_ah_slave
+	 * and we are directly creating an AV for a slave's gid_index.
+	 */
 	gid_attr = ah_attr->grh.sgid_attr;
-	if (is_vlan_dev(gid_attr->ndev))
-		vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
-	memcpy(ah->av.eth.s_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+	if (gid_attr) {
+		if (is_vlan_dev(gid_attr->ndev))
+			vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
+		memcpy(ah->av.eth.s_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+		ret = mlx4_ib_gid_index_to_real_index(ibdev, gid_attr);
+		if (ret < 0)
+			return ERR_PTR(ret);
+		ah->av.eth.gid_index = ret;
+	} else {
+		/* mlx4_ib_create_ah_slave fills in the s_mac and the vlan */
+		ah->av.eth.gid_index = ah_attr->grh.sgid_index;
+	}
 
 	if (vlan_tag < 0x1000)
 		vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
 	ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn |
 					 (rdma_ah_get_port_num(ah_attr) << 24));
-	ret = mlx4_ib_gid_index_to_real_index(ibdev, gid_attr);
-	if (ret < 0)
-		return ERR_PTR(ret);
-	ah->av.eth.gid_index = ret;
 	ah->av.eth.vlan = cpu_to_be16(vlan_tag);
 	ah->av.eth.hop_limit = grh->hop_limit;
 	if (rdma_ah_get_static_rate(ah_attr)) {
@@ -167,6 +177,40 @@ struct ib_ah *mlx4_ib_create_ah(struct i
 		return create_ib_ah(pd, ah_attr, ah); /* never fails */
 }
 
+/* AH's created via this call must be free'd by mlx4_ib_destroy_ah. */
+struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
+				      struct rdma_ah_attr *ah_attr,
+				      int slave_sgid_index, u8 *s_mac,
+				      u16 vlan_tag)
+{
+	struct rdma_ah_attr slave_attr = *ah_attr;
+	struct mlx4_ib_ah *mah;
+	struct ib_ah *ah;
+
+	slave_attr.grh.sgid_attr = NULL;
+	slave_attr.grh.sgid_index = slave_sgid_index;
+	ah = mlx4_ib_create_ah(pd, &slave_attr, NULL);
+	if (IS_ERR(ah))
+		return ah;
+
+	ah->device = pd->device;
+	ah->pd = pd;
+	ah->type = ah_attr->type;
+	mah = to_mah(ah);
+
+	/* get rid of force-loopback bit */
+	mah->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
+
+	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
+		memcpy(mah->av.eth.s_mac, s_mac, 6);
+
+	if (vlan_tag < 0x1000)
+		vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
+	mah->av.eth.vlan = cpu_to_be16(vlan_tag);
+
+	return ah;
+}
+
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 {
 	struct mlx4_ib_ah *ah = to_mah(ibah);
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -1367,13 +1367,10 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_
 	struct mlx4_mad_snd_buf *sqp_mad;
 	struct ib_ah *ah;
 	struct ib_qp *send_qp = NULL;
-	struct ib_global_route *grh;
 	unsigned wire_tx_ix = 0;
 	int ret = 0;
 	u16 wire_pkey_ix;
 	int src_qpnum;
-	u8 sgid_index;
-
 
 	sqp_ctx = dev->sriov.sqps[port-1];
 
@@ -1394,16 +1391,11 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_
 	send_qp = sqp->qp;
 
 	/* create ah */
-	grh = rdma_ah_retrieve_grh(attr);
-	sgid_index = grh->sgid_index;
-	grh->sgid_index = 0;
-	ah = rdma_create_ah(sqp_ctx->pd, attr);
+	ah = mlx4_ib_create_ah_slave(sqp_ctx->pd, attr,
+				     rdma_ah_retrieve_grh(attr)->sgid_index,
+				     s_mac, vlan_id);
 	if (IS_ERR(ah))
 		return -ENOMEM;
-	grh->sgid_index = sgid_index;
-	to_mah(ah)->av.ib.gid_index = sgid_index;
-	/* get rid of force-loopback bit */
-	to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
 	spin_lock(&sqp->tx_lock);
 	if (sqp->tx_ix_head - sqp->tx_ix_tail >=
 	    (MLX4_NUM_TUNNEL_BUFS - 1))
@@ -1445,12 +1437,6 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_
 	wr.wr.num_sge = 1;
 	wr.wr.opcode = IB_WR_SEND;
 	wr.wr.send_flags = IB_SEND_SIGNALED;
-	if (s_mac)
-		memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6);
-	if (vlan_id < 0x1000)
-		vlan_id |= (rdma_ah_get_sl(attr) & 7) << 13;
-	to_mah(ah)->av.eth.vlan = cpu_to_be16(vlan_id);
-
 
 	ret = ib_post_send(send_qp, &wr.wr, &bad_wr);
 	if (!ret)
@@ -1461,7 +1447,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_
 	spin_unlock(&sqp->tx_lock);
 	sqp->tx_ring[wire_tx_ix].ah = NULL;
 out:
-	rdma_destroy_ah(ah);
+	mlx4_ib_destroy_ah(ah);
 	return ret;
 }
 
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -760,6 +760,10 @@ void mlx4_ib_cq_clean(struct mlx4_ib_cq
 
 struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 				struct ib_udata *udata);
+struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
+				      struct rdma_ah_attr *ah_attr,
+				      int slave_sgid_index, u8 *s_mac,
+				      u16 vlan_tag);
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 int mlx4_ib_destroy_ah(struct ib_ah *ah);