From: Karsten Graul <kgraul@linux.ibm.com>
Subject: net/smc: add infrastructure to send delete rkey messages
Patch-mainline: v5.0-rc1
Git-commit: 60e03c62c5db22c5eb63bcb6ce226cf05f4ee47c
References: FATE#325698, LTC#167867, bsc#1113481
Description: net/smc: bugfix and compatibility patches
Symptom: Random hangs in smc processing:
user space application hangs in socket send() or recv() call or
does never get a notification from a select() call.
Missing compatibility to other platforms:
confirm rkey and delete rkey processing is required by the
design, but delete rkey processing is missing. This leads to
protocol failures when communicating with other platforms like
zOS. The SMC-D shutdown signal support is missing, so there is
no detection if the remote peer closed the link group.
Broken administration of available WR send payload buffers due to
a use-after-free condition.
Problem: Misbehaviour regarding the user space api can lead to hang
situations. SMC is not fully compatible to some other platforms
due to missing rkey processing and SMC-D shutdown signal support.
Solution: Fixed protocoll deficiencies by implementing the required rkey
processing. For SMC-D, the cursors are now handled atomically to
handle parallel modifications. The SMC-D shutdown signal is now
processed when received and sent to the remote peer if needed.
Prereq patches are included.
Reproduction: Run SMC on a loaded system against zOS as peer system.
Upstream-Description:
net/smc: add infrastructure to send delete rkey messages
Add the infrastructure to send LLC messages of type DELETE RKEY to
unregister a shared memory region at the peer.
Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Signed-off-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Acked-by: Petr Tesarik <ptesarik@suse.com>
---
net/smc/smc_core.h | 3 +++
net/smc/smc_llc.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
net/smc/smc_llc.h | 2 ++
3 files changed, 57 insertions(+), 1 deletion(-)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -108,6 +108,9 @@ struct smc_link {
int llc_testlink_time; /* testlink interval */
struct completion llc_confirm_rkey; /* wait 4 rx of cnf rkey */
int llc_confirm_rkey_rc; /* rc from cnf rkey msg */
+ struct completion llc_delete_rkey; /* wait 4 rx of del rkey */
+ int llc_delete_rkey_rc; /* rc from del rkey msg */
+ struct mutex llc_delete_rkey_mutex; /* serialize usage */
};
/* For now we just allow one parallel link per link group. The SMC protocol
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -237,6 +237,29 @@ static int smc_llc_send_confirm_rkey(str
return rc;
}
+/* send LLC delete rkey request */
+static int smc_llc_send_delete_rkey(struct smc_link *link,
+ struct smc_buf_desc *rmb_desc)
+{
+ struct smc_llc_msg_delete_rkey *rkeyllc;
+ struct smc_wr_tx_pend_priv *pend;
+ struct smc_wr_buf *wr_buf;
+ int rc;
+
+ rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
+ if (rc)
+ return rc;
+ rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
+ memset(rkeyllc, 0, sizeof(*rkeyllc));
+ rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
+ rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
+ rkeyllc->num_rkeys = 1;
+ rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
+ /* send llc message */
+ rc = smc_wr_tx_send(link, pend);
+ return rc;
+}
+
/* prepare an add link message */
static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
struct smc_link *link, u8 mac[], u8 gid[],
@@ -508,7 +531,9 @@ static void smc_llc_rx_delete_rkey(struc
int i, max;
if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
- /* unused as long as we don't send this type of msg */
+ link->llc_delete_rkey_rc = llc->hd.flags &
+ SMC_LLC_FLAG_RKEY_NEG;
+ complete(&link->llc_delete_rkey);
} else {
max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
for (i = 0; i < max; i++) {
@@ -609,6 +634,8 @@ int smc_llc_link_init(struct smc_link *l
init_completion(&link->llc_add);
init_completion(&link->llc_add_resp);
init_completion(&link->llc_confirm_rkey);
+ init_completion(&link->llc_delete_rkey);
+ mutex_init(&link->llc_delete_rkey_mutex);
init_completion(&link->llc_testlink_resp);
INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
return 0;
@@ -649,6 +676,7 @@ int smc_llc_do_confirm_rkey(struct smc_l
{
int rc;
+ /* protected by mutex smc_create_lgr_pending */
reinit_completion(&link->llc_confirm_rkey);
rc = smc_llc_send_confirm_rkey(link, rmb_desc);
if (rc)
@@ -661,6 +689,29 @@ int smc_llc_do_confirm_rkey(struct smc_l
return 0;
}
+/* unregister an rtoken at the remote peer */
+int smc_llc_do_delete_rkey(struct smc_link *link,
+ struct smc_buf_desc *rmb_desc)
+{
+ int rc;
+
+ mutex_lock(&link->llc_delete_rkey_mutex);
+ reinit_completion(&link->llc_delete_rkey);
+ rc = smc_llc_send_delete_rkey(link, rmb_desc);
+ if (rc)
+ goto out;
+ /* receive DELETE RKEY response from server over RoCE fabric */
+ rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey,
+ SMC_LLC_WAIT_TIME);
+ if (rc <= 0 || link->llc_delete_rkey_rc)
+ rc = -EFAULT;
+ else
+ rc = 0;
+out:
+ mutex_unlock(&link->llc_delete_rkey_mutex);
+ return rc;
+}
+
/***************************** init, exit, misc ******************************/
static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
--- a/net/smc/smc_llc.h
+++ b/net/smc/smc_llc.h
@@ -48,6 +48,8 @@ void smc_llc_link_inactive(struct smc_li
void smc_llc_link_clear(struct smc_link *link);
int smc_llc_do_confirm_rkey(struct smc_link *link,
struct smc_buf_desc *rmb_desc);
+int smc_llc_do_delete_rkey(struct smc_link *link,
+ struct smc_buf_desc *rmb_desc);
int smc_llc_init(void) __init;
#endif /* SMC_LLC_H */