From 06245c19589d412a41609ec2e2336cf568ad39c0 Mon Sep 17 00:00:00 2001
From: David Disseldorp <ddiss@suse.de>
Date: Wed, 26 Aug 2015 20:44:24 +0200
Subject: [PATCH] target/rbd: add pr_reserve support
Patch-mainline: Not yet, SES2 clustered LIO/RBD
References: fate#318836

Add a tcm_rbd_pr_ops handler for PR reserve requests. The on-disk PR
info xattr format is updated to accommodate a reservation key, IT nexus
and type, alongside existing registration information.
The on disk format change is done without a corresponding version bump,
as this version is unreleased.

Signed-off-by: David Disseldorp <ddiss@suse.de>
---
 drivers/target/target_core_rbd.c |  271 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 268 insertions(+), 3 deletions(-)

--- a/drivers/target/target_core_rbd.c
+++ b/drivers/target/target_core_rbd.c
@@ -685,8 +685,11 @@ static bool tcm_rbd_get_write_cache(stru
 #define TCM_RBD_PR_INFO_XATTR_FIELD_VER		0
 #define TCM_RBD_PR_INFO_XATTR_FIELD_SEQ		1
 #define TCM_RBD_PR_INFO_XATTR_FIELD_GEN		2
-#define TCM_RBD_PR_INFO_XATTR_FIELD_NUM_REGS	3
-#define TCM_RBD_PR_INFO_XATTR_FIELD_REGS_START	4
+#define TCM_RBD_PR_INFO_XATTR_FIELD_SCSI3_RSV	3
+#define TCM_RBD_PR_INFO_XATTR_FIELD_NUM_REGS	4
+#define TCM_RBD_PR_INFO_XATTR_FIELD_REGS_START	5
+
+#define TCM_RBD_PR_INFO_XATTR_VAL_SCSI3_RSV_ABSENT	"No SPC-3 Reservation holder"
 
 /* don't allow encoded PR info to exceed 8K */
 #define TCM_RBD_PR_INFO_XATTR_MAX_SIZE 8192
@@ -705,6 +708,21 @@ static bool tcm_rbd_get_write_cache(stru
  * string for storage within an RBD object xattr. String based storage allows
  * us to use xattr compare and write operations for atomic PR info updates.
  */
+struct tcm_rbd_pr_rsv {
+	u64 key;		/* registered key */
+	/*
+	 * I-T nexus for reservation. Separate to reg, so that all_tg_pt flag
+	 * can be supported in future.
+	 */
+	char it_nexus[TCM_RBD_PR_IT_NEXUS_MAXLEN];
+	int type;		/* PR_TYPE_... */
+	/* scope is always PR_SCOPE_LU_SCOPE */
+};
+#define TCM_RBD_PR_INFO_XATTR_ENCODED_PR_RSV_MAXLEN		\
+	((sizeof("0x") + sizeof(u64) * 2) + sizeof(" ") +	\
+	 TCM_RBD_PR_IT_NEXUS_MAXLEN + sizeof(" ") +		\
+	 (sizeof("0x") + sizeof(u64) * 2) + sizeof("\n"))
+
 struct tcm_rbd_pr_reg {
 	struct list_head regs_node;
 	u64 key;		/* registered key */
@@ -719,6 +737,7 @@ struct tcm_rbd_pr_info {
 	u32 vers;		/* on disk format version number */
 	u32 seq; 		/* sequence number bumped every xattr write */
 	u32 gen; 		/* PR generation number */
+	struct tcm_rbd_pr_rsv *rsv;	/* SCSI3 reservation if any */
 	u32 num_regs;		/* number of registrations */
 	struct list_head regs;	/* list of registrations */
 };
@@ -726,6 +745,7 @@ struct tcm_rbd_pr_info {
 	((sizeof("0x") + sizeof(u32) * 2) + sizeof("\n") +		\
 	 (sizeof("0x") + sizeof(u32) * 2) + sizeof("\n") +		\
 	 (sizeof("0x") + sizeof(u32) * 2) + sizeof("\n") +		\
+	 TCM_RBD_PR_INFO_XATTR_ENCODED_PR_RSV_MAXLEN +	 		\
 	 (sizeof("0x") + sizeof(u32) * 2) + sizeof("\n") +		\
 	 (TCM_RBD_PR_INFO_XATTR_ENCODED_PR_REG_MAXLEN * _num_regs) +	\
 	 sizeof("\0"))
@@ -766,12 +786,64 @@ tcm_rbd_pr_info_free(struct tcm_rbd_pr_i
 	struct tcm_rbd_pr_reg *reg;
 	struct tcm_rbd_pr_reg *reg_n;
 
+	kfree(pr_info->rsv);
 	list_for_each_entry_safe(reg, reg_n, &pr_info->regs, regs_node) {
 		kfree(reg);
 	}
 	kfree(pr_info);
 }
 
+static bool
+tcm_rbd_is_rsv_holder(struct tcm_rbd_pr_rsv *rsv, struct tcm_rbd_pr_reg *reg,
+		      bool *rsv_is_all_reg)
+{
+	BUG_ON(!rsv);
+	BUG_ON(!reg);
+	if ((rsv->type == PR_TYPE_WRITE_EXCLUSIVE_ALLREG)
+			|| (rsv->type == PR_TYPE_EXCLUSIVE_ACCESS_ALLREG)) {
+		/* any registeration is a reservation holder */
+		if (rsv_is_all_reg)
+			*rsv_is_all_reg = true;
+		return true;
+	}
+	if (rsv_is_all_reg)
+		*rsv_is_all_reg = false;
+
+	if ((rsv->key == reg->key)
+	 && !strncmp(rsv->it_nexus, reg->it_nexus, ARRAY_SIZE(rsv->it_nexus))) {
+		return true;
+	}
+
+	return false;
+}
+
+static int
+tcm_rbd_pr_info_rsv_set(struct tcm_rbd_pr_info *pr_info, u64 key, char *nexus,
+			int type)
+{
+	struct tcm_rbd_pr_rsv *rsv;
+
+	if (pr_info->rsv != NULL) {
+		pr_err("rsv_set called with existing reservation\n");
+		return -EINVAL;
+	}
+
+	rsv = kmalloc(sizeof(*rsv), GFP_KERNEL);
+	if (!rsv) {
+		return -ENOMEM;
+	}
+
+	rsv->key = key;
+	strlcpy(rsv->it_nexus, nexus, ARRAY_SIZE(rsv->it_nexus));
+	rsv->type = type;
+
+	pr_info->rsv = rsv;
+
+	dout("pr_info rsv set: 0x%llx %s %d\n", key, nexus, type);
+
+	return 0;
+}
+
 static int
 tcm_rbd_pr_info_append_reg(struct tcm_rbd_pr_info *pr_info, char *nexus,
 			   u64 key)
@@ -891,6 +963,39 @@ tcm_rbd_pr_info_num_regs_decode(char *st
 }
 
 static int
+tcm_rbd_pr_info_rsv_decode(char *str, struct tcm_rbd_pr_rsv **_rsv)
+{
+	struct tcm_rbd_pr_rsv *rsv;
+	int rc;
+
+	BUG_ON(!_rsv);
+	if (!strncmp(str, TCM_RBD_PR_INFO_XATTR_VAL_SCSI3_RSV_ABSENT,
+		    sizeof(TCM_RBD_PR_INFO_XATTR_VAL_SCSI3_RSV_ABSENT))) {
+		/* no reservation. Ensure pr_info->rsv is NULL */
+		rsv = NULL;
+	} else {
+		rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
+		if (!rsv) {
+			return -ENOMEM;
+		}
+
+		/* reservation key, I-T nexus, and type with space separators */
+		rc = sscanf(str, "0x%016llx %"
+			    __stringify(TCM_RBD_PR_IT_NEXUS_MAXLEN)
+			    "s 0x%08x", &rsv->key, rsv->it_nexus, &rsv->type);
+		if (rc != 3) {
+			pr_err("failed to parse PR rsv: %s\n", str);
+			kfree(rsv);
+			return -EINVAL;
+		}
+	}
+
+	dout("processed pr_info rsv: %s\n", str);
+	*_rsv = rsv;
+	return 0;
+}
+
+static int
 tcm_rbd_pr_info_reg_decode(char *str, struct tcm_rbd_pr_reg **_reg)
 {
 	struct tcm_rbd_pr_reg *reg;
@@ -971,6 +1076,11 @@ tcm_rbd_pr_info_decode(char *pr_xattr,
 			if (rc < 0) {
 				goto err_info_free;
 			}
+		} else if (field == TCM_RBD_PR_INFO_XATTR_FIELD_SCSI3_RSV) {
+			rc = tcm_rbd_pr_info_rsv_decode(str, &pr_info->rsv);
+			if (rc < 0) {
+				goto err_info_free;
+			}
 		} else if (field == TCM_RBD_PR_INFO_XATTR_FIELD_NUM_REGS) {
 			rc = tcm_rbd_pr_info_num_regs_decode(str,
 							    &pr_info->num_regs);
@@ -1032,6 +1142,28 @@ tcm_rbd_pr_info_vers_seq_gen_encode(char
 }
 
 static int
+tcm_rbd_pr_info_rsv_encode(char *buf, size_t buf_remain,
+			   struct tcm_rbd_pr_rsv *rsv)
+{
+	int rc;
+
+	if (!rsv) {
+		/* no reservation */
+		rc = snprintf(buf, buf_remain, "%s\n",
+			      TCM_RBD_PR_INFO_XATTR_VAL_SCSI3_RSV_ABSENT);
+	} else {
+		rc = snprintf(buf, buf_remain, "0x%016llx %s 0x%08x\n",
+			      rsv->key, rsv->it_nexus, rsv->type);
+	}
+	if ((rc < 0) || (rc >= buf_remain)) {
+		pr_err("failed to encode PR reservation\n");
+		return -EINVAL;
+	}
+
+	return rc;
+}
+
+static int
 tcm_rbd_pr_info_num_regs_encode(char *buf, size_t buf_remain, u32 num_regs)
 {
 	int rc;
@@ -1103,6 +1235,15 @@ tcm_rbd_pr_info_encode(struct tcm_rbd_pr
 	p += rc;
 	buf_remain -= rc;
 
+	rc = tcm_rbd_pr_info_rsv_encode(p, buf_remain, pr_info->rsv);
+	 if (rc < 0) {
+		rc = -EINVAL;
+		goto err_xattr_free;
+	}
+
+	p += rc;
+	buf_remain -= rc;
+
 	rc = tcm_rbd_pr_info_num_regs_encode(p, buf_remain, pr_info->num_regs);
 	if (rc < 0) {
 		rc = -EINVAL;
@@ -1467,6 +1608,11 @@ tcm_rbd_execute_pr_register(struct se_cm
 	sense_reason_t ret;
 	int retries = 0;
 
+	if (!cmd->se_sess || !cmd->se_lun) {
+		pr_err("SPC-3 PR: se_sess || struct se_lun is NULL!\n");
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+	}
+
 	if (!aptpl) {
 		/*
 		 * Currently unsupported by block layer API (hch):
@@ -1510,7 +1656,7 @@ retry:
 	/* check for an existing registration */
 	existing_reg = NULL;
 	list_for_each_entry(reg, &pr_info->regs, regs_node) {
-		if (!strcmp(reg->it_nexus, nexus_buf)) {
+		if (!strncmp(reg->it_nexus, nexus_buf, ARRAY_SIZE(nexus_buf))) {
 			dout("found existing PR reg for %s\n", nexus_buf);
 			existing_reg = reg;
 			break;
@@ -1582,10 +1728,129 @@ err_out:
 	return ret;
 }
 
+static sense_reason_t
+tcm_rbd_execute_pr_reserve(struct se_cmd *cmd, int type, u64 key)
+{
+	struct se_device *dev = cmd->se_dev;
+	struct tcm_rbd_dev *tcm_rbd_dev = TCM_RBD_DEV(dev);
+	char nexus_buf[TCM_RBD_PR_IT_NEXUS_MAXLEN];
+	struct tcm_rbd_pr_info *pr_info;
+	struct tcm_rbd_pr_reg *reg;
+	struct tcm_rbd_pr_reg *existing_reg;
+	char *pr_xattr;
+	int pr_xattr_len;
+	int rc;
+	sense_reason_t ret;
+	int retries = 0;
+
+	if (!cmd->se_sess || !cmd->se_lun) {
+		pr_err("SPC-3 PR: se_sess || struct se_lun is NULL!\n");
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+	}
+
+	rc = tcm_rbd_gen_it_nexus(cmd->se_sess, nexus_buf,
+				  ARRAY_SIZE(nexus_buf));
+	if (rc < 0)
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+
+retry:
+	pr_info = NULL;
+	pr_xattr = NULL;
+	pr_xattr_len = 0;
+	rc = tcm_rbd_pr_info_get(tcm_rbd_dev, &pr_info, &pr_xattr,
+				 &pr_xattr_len);
+	if (rc < 0) {
+		/* existing registration required for reservation */
+		pr_err("failed to obtain PR info\n");
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+	}
+
+	/* check for an existing registration */
+	existing_reg = NULL;
+	list_for_each_entry(reg, &pr_info->regs, regs_node) {
+		if (!strncmp(reg->it_nexus, nexus_buf, ARRAY_SIZE(nexus_buf))) {
+			dout("found existing PR reg for %s\n", nexus_buf);
+			existing_reg = reg;
+			break;
+		}
+	}
+
+	if (!existing_reg) {
+		pr_err("SPC-3 PR: Unable to locate registration for RESERVE\n");
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+	}
+
+	if (key != existing_reg->key) {
+		pr_err("SPC-3 PR RESERVE: Received res_key: 0x%016Lx"
+			" does not match existing SA REGISTER res_key:"
+			" 0x%016Lx\n", key, existing_reg->key);
+		ret = TCM_RESERVATION_CONFLICT;
+		goto err_info_free;
+	}
+
+	if (pr_info->rsv) {
+		if (!tcm_rbd_is_rsv_holder(pr_info->rsv, existing_reg, NULL)) {
+			pr_err("SPC-3 PR: Attempted RESERVE from %s while"
+			       " reservation already held by %s, returning"
+			       " RESERVATION_CONFLICT\n",
+			       nexus_buf, pr_info->rsv->it_nexus);
+			ret = TCM_RESERVATION_CONFLICT;
+			goto err_info_free;
+		}
+
+		if (pr_info->rsv->type != type) {
+			/* scope already checked */
+			pr_err("SPC-3 PR: Attempted RESERVE from %s trying to "
+			       "change TYPE, returning RESERVATION_CONFLICT\n",
+			       existing_reg->it_nexus);
+			ret = TCM_RESERVATION_CONFLICT;
+			goto err_info_free;
+		}
+
+		dout("reserve matches existing reservation, nothing to do\n");
+		goto done;
+	}
+
+	/* new reservation */
+	rc = tcm_rbd_pr_info_rsv_set(pr_info, key, nexus_buf, type);
+	if (rc < 0) {
+		pr_err("failed to set PR info reservation\n");
+		ret = TCM_OUT_OF_RESOURCES;
+		goto err_info_free;
+	}
+
+	rc = tcm_rbd_pr_info_replace(tcm_rbd_dev, pr_xattr, pr_xattr_len,
+				     pr_info);
+	if (rc == -ECANCELED) {
+		pr_warn("atomic PR info update failed due to parallel "
+			"change, expected(%d) %s. Retrying...\n",
+			pr_xattr_len, pr_xattr);
+		retries++;
+		if (retries <= TCM_RBD_PR_REG_MAX_RETRIES) {
+			tcm_rbd_pr_info_free(pr_info);
+			kfree(pr_xattr);
+			goto retry;
+		}
+	}
+	if (rc < 0) {
+		pr_err("atomic PR info update failed: %d\n", rc);
+		ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+		goto err_info_free;
+	}
+
+done:
+	ret = TCM_NO_SENSE;
+err_info_free:
+	tcm_rbd_pr_info_free(pr_info);
+	kfree(pr_xattr);
+	return ret;
+}
+
 static struct target_pr_ops tcm_rbd_pr_ops = {
 	.pr_read_keys		= tcm_rbd_execute_pr_read_keys,
 
 	.pr_register		= tcm_rbd_execute_pr_register,
+	.pr_reserve		= tcm_rbd_execute_pr_reserve,
 };
 
 static const struct target_backend_ops tcm_rbd_ops = {