From: Sebastian Sanchez <sebastian.sanchez@intel.com>
Date: Thu, 1 Feb 2018 10:46:31 -0800
Subject: IB/hfi1: Look up ibport using a pointer in receive path
Patch-mainline: v4.16-rc1
Git-commit: bdaf96f6500880401fe692cd0bf57afb596b135c
References: bsc#1096793 FATE#325050
In the receive path, hfi1_ibport is looked up by indexing into an
array. A profile shows this to be expensive. The receive context
data has a pointer to the ibport data, use that pointer instead.
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Sebastian Sanchez <sebastian.sanchez@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/infiniband/hw/hfi1/driver.c | 14 ++++++++------
drivers/infiniband/hw/hfi1/rc.c | 36 +++++++++++++++++++++---------------
drivers/infiniband/hw/hfi1/verbs.h | 3 +--
3 files changed, 30 insertions(+), 23 deletions(-)
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -650,9 +650,10 @@ next:
}
}
-static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd)
+static void process_rcv_qp_work(struct hfi1_packet *packet)
{
struct rvt_qp *qp, *nqp;
+ struct hfi1_ctxtdata *rcd = packet->rcd;
/*
* Iterate over all QPs waiting to respond.
@@ -662,7 +663,8 @@ static void process_rcv_qp_work(struct h
list_del_init(&qp->rspwait);
if (qp->r_flags & RVT_R_RSP_NAK) {
qp->r_flags &= ~RVT_R_RSP_NAK;
- hfi1_send_rc_ack(rcd, qp, 0);
+ packet->qp = qp;
+ hfi1_send_rc_ack(packet, 0);
}
if (qp->r_flags & RVT_R_RSP_SEND) {
unsigned long flags;
@@ -683,7 +685,7 @@ static noinline int max_packet_exceeded(
if (thread) {
if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
/* allow defered processing */
- process_rcv_qp_work(packet->rcd);
+ process_rcv_qp_work(packet);
cond_resched();
return RCV_PKT_OK;
} else {
@@ -825,7 +827,7 @@ int handle_receive_interrupt_nodma_rtail
last = RCV_PKT_DONE;
process_rcv_update(last, &packet);
}
- process_rcv_qp_work(rcd);
+ process_rcv_qp_work(&packet);
rcd->head = packet.rhqoff;
bail:
finish_packet(&packet);
@@ -854,7 +856,7 @@ int handle_receive_interrupt_dma_rtail(s
last = RCV_PKT_DONE;
process_rcv_update(last, &packet);
}
- process_rcv_qp_work(rcd);
+ process_rcv_qp_work(&packet);
rcd->head = packet.rhqoff;
bail:
finish_packet(&packet);
@@ -1084,7 +1086,7 @@ int handle_receive_interrupt(struct hfi1
process_rcv_update(last, &packet);
}
- process_rcv_qp_work(rcd);
+ process_rcv_qp_work(&packet);
rcd->head = packet.rhqoff;
bail:
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -728,14 +728,16 @@ static inline void hfi1_make_bth_aeth(st
ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
}
-static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn)
+static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn)
{
- struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+ struct rvt_qp *qp = packet->qp;
+ struct hfi1_ibport *ibp;
unsigned long flags;
spin_lock_irqsave(&qp->s_lock, flags);
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
goto unlock;
+ ibp = rcd_to_iport(packet->rcd);
this_cpu_inc(*ibp->rvp.rc_qacks);
qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
qp->s_nak_state = qp->r_nak_state;
@@ -749,13 +751,14 @@ unlock:
spin_unlock_irqrestore(&qp->s_lock, flags);
}
-static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp,
+static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet,
struct hfi1_opa_header *opa_hdr,
u8 sc5, bool is_fecn,
u64 *pbc_flags, u32 *hwords,
u32 *nwords)
{
- struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+ struct rvt_qp *qp = packet->qp;
+ struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
struct ib_header *hdr = &opa_hdr->ibh;
struct ib_other_headers *ohdr;
@@ -796,13 +799,14 @@ static inline void hfi1_make_rc_ack_9B(s
hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
}
-static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp,
+static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet,
struct hfi1_opa_header *opa_hdr,
u8 sc5, bool is_fecn,
u64 *pbc_flags, u32 *hwords,
u32 *nwords)
{
- struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+ struct rvt_qp *qp = packet->qp;
+ struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
struct hfi1_16b_header *hdr = &opa_hdr->opah;
struct ib_other_headers *ohdr;
@@ -848,7 +852,7 @@ static inline void hfi1_make_rc_ack_16B(
hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
}
-typedef void (*hfi1_make_rc_ack)(struct rvt_qp *qp,
+typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet,
struct hfi1_opa_header *opa_hdr,
u8 sc5, bool is_fecn,
u64 *pbc_flags, u32 *hwords,
@@ -868,9 +872,10 @@ static const hfi1_make_rc_ack hfi1_make_
* Note that RDMA reads and atomics are handled in the
* send side QP state and send engine.
*/
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
- struct rvt_qp *qp, bool is_fecn)
+void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn)
{
+ struct hfi1_ctxtdata *rcd = packet->rcd;
+ struct rvt_qp *qp = packet->qp;
struct hfi1_ibport *ibp = rcd_to_iport(rcd);
struct hfi1_qp_priv *priv = qp->priv;
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
@@ -887,13 +892,13 @@ void hfi1_send_rc_ack(struct hfi1_ctxtda
/* Don't send ACK or NAK if a RDMA read or atomic is pending. */
if (qp->s_flags & RVT_S_RESP_PENDING) {
- hfi1_queue_rc_ack(qp, is_fecn);
+ hfi1_queue_rc_ack(packet, is_fecn);
return;
}
/* Ensure s_rdma_ack_cnt changes are committed */
if (qp->s_rdma_ack_cnt) {
- hfi1_queue_rc_ack(qp, is_fecn);
+ hfi1_queue_rc_ack(packet, is_fecn);
return;
}
@@ -902,7 +907,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtda
return;
/* Make the appropriate header */
- hfi1_make_rc_ack_tbl[priv->hdr_type](qp, &opa_hdr, sc5, is_fecn,
+ hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn,
&pbc_flags, &hwords, &nwords);
plen = 2 /* PBC */ + hwords + nwords;
@@ -916,7 +921,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtda
* so that when enough buffer space becomes available,
* the ACK is sent ahead of other outgoing packets.
*/
- hfi1_queue_rc_ack(qp, is_fecn);
+ hfi1_queue_rc_ack(packet, is_fecn);
return;
}
trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
@@ -1534,7 +1539,7 @@ static void rc_rcv_resp(struct hfi1_pack
void *data = packet->payload;
u32 tlen = packet->tlen;
struct rvt_qp *qp = packet->qp;
- struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+ struct hfi1_ibport *ibp;
struct ib_other_headers *ohdr = packet->ohdr;
struct rvt_swqe *wqe;
enum ib_wc_status status;
@@ -1691,6 +1696,7 @@ ack_op_err:
goto ack_err;
ack_seq_err:
+ ibp = rcd_to_iport(rcd);
rdma_seq_err(qp, ibp, psn, rcd);
goto ack_done;
@@ -2472,7 +2478,7 @@ nack_acc:
qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
qp->r_ack_psn = qp->r_psn;
send_ack:
- hfi1_send_rc_ack(rcd, qp, is_fecn);
+ hfi1_send_rc_ack(packet, is_fecn);
}
void hfi1_rc_hdrerr(
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -358,8 +358,7 @@ void hfi1_do_send(struct rvt_qp *qp, boo
void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
enum ib_wc_status status);
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
- bool is_fecn);
+void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn);
int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);