From: Steve Wise <swise@opengridcomputing.com>
Date: Thu, 1 Mar 2018 13:58:13 -0800
Subject: RDMA/nldev: provide detailed MR information
Patch-mainline: v4.17-rc1
Git-commit: fccec5b89ac61ebe2f353feecd08a16621f2418b
References: bsc#1103992 FATE#326009
Implement the RDMA nldev netlink interface for dumping detailed
MR information.
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/infiniband/core/nldev.c | 56 +++++++++++++++++++++++++++++++++++
drivers/infiniband/core/restrack.c | 36 ++++++++++++++--------
drivers/infiniband/core/uverbs_cmd.c | 2 +
drivers/infiniband/core/verbs.c | 3 +
include/rdma/ib_verbs.h | 5 +++
include/rdma/restrack.h | 4 ++
include/uapi/rdma/rdma_netlink.h | 9 +++++
7 files changed, 102 insertions(+), 13 deletions(-)
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -85,6 +85,12 @@ static const struct nla_policy nldev_pol
[RDMA_NLDEV_ATTR_RES_CQE] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 },
[RDMA_NLDEV_ATTR_RES_POLL_CTX] = { .type = NLA_U8 },
+ [RDMA_NLDEV_ATTR_RES_MR] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = { .type = NLA_NESTED },
+ [RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_LKEY] = { .type = NLA_U32 },
+ [RDMA_NLDEV_ATTR_RES_IOVA] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_RES_MRLEN] = { .type = NLA_U64 },
};
static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
@@ -197,6 +203,7 @@ static int fill_res_info(struct sk_buff
[RDMA_RESTRACK_CQ] = "cq",
[RDMA_RESTRACK_QP] = "qp",
[RDMA_RESTRACK_CM_ID] = "cm_id",
+ [RDMA_RESTRACK_MR] = "mr",
};
struct rdma_restrack_root *res = &device->res;
@@ -397,6 +404,41 @@ out:
return -EMSGSIZE;
}
+static int fill_res_mr_entry(struct sk_buff *msg, struct netlink_callback *cb,
+ struct rdma_restrack_entry *res, uint32_t port)
+{
+ struct ib_mr *mr = container_of(res, struct ib_mr, res);
+ struct nlattr *entry_attr;
+
+ entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_MR_ENTRY);
+ if (!entry_attr)
+ goto out;
+
+ if (netlink_capable(cb->skb, CAP_NET_ADMIN)) {
+ if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
+ goto err;
+ if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
+ goto err;
+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_IOVA,
+ mr->iova, 0))
+ goto err;
+ }
+
+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length, 0))
+ goto err;
+
+ if (fill_res_name_pid(msg, res))
+ goto err;
+
+ nla_nest_end(msg, entry_attr);
+ return 0;
+
+err:
+ nla_nest_cancel(msg, entry_attr);
+out:
+ return -EMSGSIZE;
+}
+
static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack)
{
@@ -694,6 +736,11 @@ static const struct nldev_fill_res_entry
.nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET,
.nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
},
+ [RDMA_RESTRACK_MR] = {
+ .fill_res_func = fill_res_mr_entry,
+ .nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET,
+ .nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
+ },
};
static int res_get_common_dumpit(struct sk_buff *skb,
@@ -848,6 +895,12 @@ static int nldev_res_get_cq_dumpit(struc
return res_get_common_dumpit(skb, cb, RDMA_RESTRACK_CQ);
}
+static int nldev_res_get_mr_dumpit(struct sk_buff *skb,
+ struct netlink_callback *cb)
+{
+ return res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR);
+}
+
static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
[RDMA_NLDEV_CMD_GET] = {
.doit = nldev_get_doit,
@@ -880,6 +933,9 @@ static const struct rdma_nl_cbs nldev_cb
[RDMA_NLDEV_CMD_RES_CQ_GET] = {
.dump = nldev_res_get_cq_dumpit,
},
+ [RDMA_NLDEV_CMD_RES_MR_GET] = {
+ .dump = nldev_res_get_mr_dumpit,
+ },
};
void __init nldev_init(void)
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -43,22 +43,28 @@ EXPORT_SYMBOL(rdma_restrack_count);
static void set_kern_name(struct rdma_restrack_entry *res)
{
- enum rdma_restrack_type type = res->type;
- struct ib_qp *qp;
+ struct ib_pd *pd;
- if (type != RDMA_RESTRACK_QP)
- /* Other types already have this name embedded in */
- return;
-
- qp = container_of(res, struct ib_qp, res);
- if (!qp->pd) {
- WARN_ONCE(true, "XRC QPs are not supported\n");
- /* Survive, despite the programmer's error */
- res->kern_name = " ";
- return;
+ switch (res->type) {
+ case RDMA_RESTRACK_QP:
+ pd = container_of(res, struct ib_qp, res)->pd;
+ if (!pd) {
+ WARN_ONCE(true, "XRC QPs are not supported\n");
+ /* Survive, despite the programmer's error */
+ res->kern_name = " ";
+ }
+ break;
+ case RDMA_RESTRACK_MR:
+ pd = container_of(res, struct ib_mr, res)->pd;
+ break;
+ default:
+ /* Other types set kern_name directly */
+ pd = NULL;
+ break;
}
- res->kern_name = qp->pd->res.kern_name;
+ if (pd)
+ res->kern_name = pd->res.kern_name;
}
static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
@@ -73,6 +79,8 @@ static struct ib_device *res_to_dev(stru
case RDMA_RESTRACK_CM_ID:
return container_of(res, struct rdma_id_private,
res)->id.device;
+ case RDMA_RESTRACK_MR:
+ return container_of(res, struct ib_mr, res)->device;
default:
WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
return NULL;
@@ -90,6 +98,8 @@ static bool res_is_user(struct rdma_rest
return container_of(res, struct ib_qp, res)->uobject;
case RDMA_RESTRACK_CM_ID:
return !res->kern_name;
+ case RDMA_RESTRACK_MR:
+ return container_of(res, struct ib_mr, res)->pd->uobject;
default:
WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
return false;
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -693,6 +693,8 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverb
mr->pd = pd;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
+ mr->res.type = RDMA_RESTRACK_MR;
+ rdma_restrack_add(&mr->res);
uobj->object = mr;
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -1623,6 +1623,7 @@ int ib_dereg_mr(struct ib_mr *mr)
struct ib_pd *pd = mr->pd;
int ret;
+ rdma_restrack_del(&mr->res);
ret = mr->device->dereg_mr(mr);
if (!ret)
atomic_dec(&pd->usecnt);
@@ -1659,6 +1660,8 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *
mr->uobject = NULL;
atomic_inc(&pd->usecnt);
mr->need_inval = false;
+ mr->res.type = RDMA_RESTRACK_MR;
+ rdma_restrack_add(&mr->res);
}
return mr;
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1768,6 +1768,11 @@ struct ib_mr {
struct ib_uobject *uobject; /* user */
struct list_head qp_entry; /* FR */
};
+
+ /*
+ * Implementation details of the RDMA core, don't use in drivers:
+ */
+ struct rdma_restrack_entry res;
};
struct ib_mw {
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -34,6 +34,10 @@ enum rdma_restrack_type {
*/
RDMA_RESTRACK_CM_ID,
/**
+ * @RDMA_RESTRACK_MR: Memory Region (MR)
+ */
+ RDMA_RESTRACK_MR,
+ /**
* @RDMA_RESTRACK_MAX: Last entry, used for array dclarations
*/
RDMA_RESTRACK_MAX
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -241,6 +241,8 @@ enum rdma_nldev_command {
RDMA_NLDEV_CMD_RES_CQ_GET, /* can dump */
+ RDMA_NLDEV_CMD_RES_MR_GET, /* can dump */
+
RDMA_NLDEV_NUM_OPS
};
@@ -371,6 +373,13 @@ enum rdma_nldev_attr {
RDMA_NLDEV_ATTR_RES_USECNT, /* u64 */
RDMA_NLDEV_ATTR_RES_POLL_CTX, /* u8 */
+ RDMA_NLDEV_ATTR_RES_MR, /* nested table */
+ RDMA_NLDEV_ATTR_RES_MR_ENTRY, /* nested table */
+ RDMA_NLDEV_ATTR_RES_RKEY, /* u32 */
+ RDMA_NLDEV_ATTR_RES_LKEY, /* u32 */
+ RDMA_NLDEV_ATTR_RES_IOVA, /* u64 */
+ RDMA_NLDEV_ATTR_RES_MRLEN, /* u64 */
+
RDMA_NLDEV_ATTR_MAX
};
#endif /* _UAPI_RDMA_NETLINK_H */