From: Harald Freudenberger <freude@linux.ibm.com>
Date: Thu, 2 Jul 2020 11:22:01 +0200
Subject: s390/ap: split ap queue state machine state from device state
Git-commit: 0b641cbd24445e56073c69dd046be488dcf1965b
Patch-mainline: v5.10-rc1
References: jsc#SLE-13815 bsc#1178402 LTC#186349
The state machine for each ap queue covered a mixture of
device states and state machine (firmware queue state) states.
This patch splits the device states and the state machine
states into two different enums and variables. The major
state is the device state with currently these values:
AP_DEV_STATE_UNINITIATED - fresh and virgin, not touched
AP_DEV_STATE_OPERATING - queue dev is working normal
AP_DEV_STATE_SHUTDOWN - remove/unbind/shutdown in progress
AP_DEV_STATE_ERROR - device is in error state
only when the device state is > UNINITIATED the state machine
is run. The state machine represents the states of the firmware
queue:
AP_SM_STATE_RESET_START - starting point, reset (RAPQ) ap queue
AP_SM_STATE_RESET_WAIT - reset triggered, waiting to be finished
if irqs enabled, set up irq (AQIC)
AP_SM_STATE_SETIRQ_WAIT - enable irq triggered, waiting to be
finished, then go to IDLE
AP_SM_STATE_IDLE - queue is operational but empty
AP_SM_STATE_WORKING - queue is operational, requests are stored
and replies may wait for getting fetched
AP_SM_STATE_QUEUE_FULL - firmware queue is full, so only replies
can get fetched
For debugging each ap queue shows a sysfs attribute 'states' which
displays the device and state machine state and is only available
when the kernel is build with CONFIG_ZCRYPT_DEBUG enabled.
Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
[ SLE15-SP3 still contains power management code, which must be also
updated by this patch. ]
Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
Acked-by: Petr Tesarik <ptesarik@suse.com>
---
drivers/s390/crypto/ap_bus.c | 2
drivers/s390/crypto/ap_bus.h | 21 +++-
drivers/s390/crypto/ap_queue.c | 157 ++++++++++++++++++++++++---------
drivers/s390/crypto/zcrypt_msgtype50.c | 8 +
drivers/s390/crypto/zcrypt_msgtype6.c | 26 +++--
5 files changed, 156 insertions(+), 58 deletions(-)
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -1495,7 +1495,7 @@ static void _ap_scan_bus_adapter(int id)
if (dev) {
if (!broken) {
spin_lock_bh(&aq->lock);
- broken = aq->sm_state == AP_SM_STATE_BORKED;
+ broken = aq->dev_state == AP_DEV_STATE_ERROR;
spin_unlock_bh(&aq->lock);
}
if (broken) {
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -86,16 +86,13 @@ static inline int ap_test_bit(unsigned i
* AP queue state machine states
*/
enum ap_sm_state {
- AP_SM_STATE_RESET_START,
+ AP_SM_STATE_RESET_START = 0,
AP_SM_STATE_RESET_WAIT,
AP_SM_STATE_SETIRQ_WAIT,
AP_SM_STATE_IDLE,
AP_SM_STATE_WORKING,
AP_SM_STATE_QUEUE_FULL,
AP_SM_STATE_SUSPEND_WAIT,
- AP_SM_STATE_REMOVE, /* about to be removed from driver */
- AP_SM_STATE_UNBOUND, /* momentary not bound to a driver */
- AP_SM_STATE_BORKED, /* broken */
NR_AP_SM_STATES
};
@@ -119,6 +116,17 @@ enum ap_sm_wait {
NR_AP_SM_WAIT
};
+/*
+ * AP queue device states
+ */
+enum ap_dev_state {
+ AP_DEV_STATE_UNINITIATED = 0, /* fresh and virgin, not touched */
+ AP_DEV_STATE_OPERATING, /* queue dev is working normal */
+ AP_DEV_STATE_SHUTDOWN, /* remove/unbind/shutdown in progress */
+ AP_DEV_STATE_ERROR, /* device is in error state */
+ NR_AP_DEV_STATES
+};
+
struct ap_device;
struct ap_message;
@@ -172,10 +180,10 @@ struct ap_queue {
struct ap_card *card; /* Ptr to assoc. AP card. */
spinlock_t lock; /* Per device lock. */
void *private; /* ap driver private pointer. */
+ enum ap_dev_state dev_state; /* queue device state */
ap_qid_t qid; /* AP queue id. */
int interrupt; /* indicate if interrupts are enabled */
int queue_count; /* # messages currently on AP queue. */
- enum ap_sm_state sm_state; /* ap queue state machine state */
int pendingq_count; /* # requests on pendingq list. */
int requestq_count; /* # requests on requestq list. */
u64 total_request_count; /* # requests ever for this AP device.*/
@@ -184,6 +192,7 @@ struct ap_queue {
struct list_head pendingq; /* List of message sent to AP queue. */
struct list_head requestq; /* List of message yet to be sent. */
struct ap_message *reply; /* Per device reply message. */
+ enum ap_sm_state sm_state; /* ap queue state machine state */
};
#define to_ap_queue(x) container_of((x), struct ap_queue, ap_dev.device)
@@ -237,7 +246,7 @@ int ap_recv(ap_qid_t, unsigned long long
enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event);
enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event);
-void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg);
+int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg);
void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg);
void ap_flush_queue(struct ap_queue *aq);
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -195,7 +195,7 @@ static enum ap_sm_wait ap_sm_read(struct
aq->sm_state = AP_SM_STATE_IDLE;
return AP_SM_WAIT_NONE;
default:
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
return AP_SM_WAIT_NONE;
}
}
@@ -270,7 +270,7 @@ static enum ap_sm_wait ap_sm_write(struc
ap_msg->receive(aq, ap_msg, NULL);
return AP_SM_WAIT_AGAIN;
default:
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
return AP_SM_WAIT_NONE;
}
}
@@ -309,7 +309,7 @@ static enum ap_sm_wait ap_sm_reset(struc
case AP_RESPONSE_DECONFIGURED:
case AP_RESPONSE_CHECKSTOPPED:
default:
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
return AP_SM_WAIT_NONE;
}
}
@@ -348,7 +348,7 @@ static enum ap_sm_wait ap_sm_reset_wait(
case AP_RESPONSE_DECONFIGURED:
case AP_RESPONSE_CHECKSTOPPED:
default:
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
return AP_SM_WAIT_NONE;
}
}
@@ -385,7 +385,7 @@ static enum ap_sm_wait ap_sm_setirq_wait
case AP_RESPONSE_NO_PENDING_REPLY:
return AP_SM_WAIT_TIMEOUT;
default:
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
return AP_SM_WAIT_NONE;
}
}
@@ -422,23 +422,14 @@ static ap_func_t *ap_jumptable[NR_AP_SM_
[AP_SM_EVENT_POLL] = ap_sm_suspend_read,
[AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
},
- [AP_SM_STATE_REMOVE] = {
- [AP_SM_EVENT_POLL] = ap_sm_nop,
- [AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
- },
- [AP_SM_STATE_UNBOUND] = {
- [AP_SM_EVENT_POLL] = ap_sm_nop,
- [AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
- },
- [AP_SM_STATE_BORKED] = {
- [AP_SM_EVENT_POLL] = ap_sm_nop,
- [AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
- },
};
enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event)
{
- return ap_jumptable[aq->sm_state][event](aq);
+ if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+ return ap_jumptable[aq->sm_state][event](aq);
+ else
+ return AP_SM_WAIT_NONE;
}
enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event)
@@ -462,7 +453,7 @@ void ap_queue_suspend(struct ap_device *
aq->sm_state = AP_SM_STATE_SUSPEND_WAIT;
while (ap_sm_event(aq, AP_SM_EVENT_POLL) != AP_SM_WAIT_NONE)
;
- aq->sm_state = AP_SM_STATE_BORKED;
+ aq->dev_state = AP_DEV_STATE_ERROR;
spin_unlock_bh(&aq->lock);
}
EXPORT_SYMBOL(ap_queue_suspend);
@@ -480,12 +471,20 @@ static ssize_t request_count_show(struct
char *buf)
{
struct ap_queue *aq = to_ap_queue(dev);
+ bool valid = false;
u64 req_cnt;
spin_lock_bh(&aq->lock);
- req_cnt = aq->total_request_count;
+ if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
+ req_cnt = aq->total_request_count;
+ valid = true;
+ }
spin_unlock_bh(&aq->lock);
- return scnprintf(buf, PAGE_SIZE, "%llu\n", req_cnt);
+
+ if (valid)
+ return scnprintf(buf, PAGE_SIZE, "%llu\n", req_cnt);
+ else
+ return scnprintf(buf, PAGE_SIZE, "-\n");
}
static ssize_t request_count_store(struct device *dev,
@@ -510,7 +509,8 @@ static ssize_t requestq_count_show(struc
unsigned int reqq_cnt = 0;
spin_lock_bh(&aq->lock);
- reqq_cnt = aq->requestq_count;
+ if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+ reqq_cnt = aq->requestq_count;
spin_unlock_bh(&aq->lock);
return scnprintf(buf, PAGE_SIZE, "%d\n", reqq_cnt);
}
@@ -524,7 +524,8 @@ static ssize_t pendingq_count_show(struc
unsigned int penq_cnt = 0;
spin_lock_bh(&aq->lock);
- penq_cnt = aq->pendingq_count;
+ if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+ penq_cnt = aq->pendingq_count;
spin_unlock_bh(&aq->lock);
return scnprintf(buf, PAGE_SIZE, "%d\n", penq_cnt);
}
@@ -593,12 +594,79 @@ static ssize_t interrupt_show(struct dev
static DEVICE_ATTR_RO(interrupt);
+#ifdef CONFIG_ZCRYPT_DEBUG
+static ssize_t states_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct ap_queue *aq = to_ap_queue(dev);
+ int rc = 0;
+
+ spin_lock_bh(&aq->lock);
+ /* queue device state */
+ switch (aq->dev_state) {
+ case AP_DEV_STATE_UNINITIATED:
+ rc = scnprintf(buf, PAGE_SIZE, "UNINITIATED\n");
+ break;
+ case AP_DEV_STATE_OPERATING:
+ rc = scnprintf(buf, PAGE_SIZE, "OPERATING");
+ break;
+ case AP_DEV_STATE_SHUTDOWN:
+ rc = scnprintf(buf, PAGE_SIZE, "SHUTDOWN");
+ break;
+ case AP_DEV_STATE_ERROR:
+ rc = scnprintf(buf, PAGE_SIZE, "ERROR");
+ break;
+ default:
+ rc = scnprintf(buf, PAGE_SIZE, "UNKNOWN");
+ }
+ /* state machine state */
+ if (aq->dev_state) {
+ switch (aq->sm_state) {
+ case AP_SM_STATE_RESET_START:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [RESET_START]\n");
+ break;
+ case AP_SM_STATE_RESET_WAIT:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [RESET_WAIT]\n");
+ break;
+ case AP_SM_STATE_SETIRQ_WAIT:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [SETIRQ_WAIT]\n");
+ break;
+ case AP_SM_STATE_IDLE:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [IDLE]\n");
+ break;
+ case AP_SM_STATE_WORKING:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [WORKING]\n");
+ break;
+ case AP_SM_STATE_QUEUE_FULL:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [FULL]\n");
+ break;
+ default:
+ rc += scnprintf(buf + rc, PAGE_SIZE - rc,
+ " [UNKNOWN]\n");
+ }
+ }
+ spin_unlock_bh(&aq->lock);
+
+ return rc;
+}
+static DEVICE_ATTR_RO(states);
+#endif
+
static struct attribute *ap_queue_dev_attrs[] = {
&dev_attr_request_count.attr,
&dev_attr_requestq_count.attr,
&dev_attr_pendingq_count.attr,
&dev_attr_reset.attr,
&dev_attr_interrupt.attr,
+#ifdef CONFIG_ZCRYPT_DEBUG
+ &dev_attr_states.attr,
+#endif
NULL
};
@@ -638,7 +706,6 @@ struct ap_queue *ap_queue_create(ap_qid_
aq->ap_dev.device.type = &ap_queue_type;
aq->ap_dev.device_type = device_type;
aq->qid = qid;
- aq->sm_state = AP_SM_STATE_UNBOUND;
aq->interrupt = AP_INTR_DISABLED;
spin_lock_init(&aq->lock);
INIT_LIST_HEAD(&aq->pendingq);
@@ -663,22 +730,30 @@ EXPORT_SYMBOL(ap_queue_init_reply);
* @aq: The AP device to queue the message to
* @ap_msg: The message that is to be added
*/
-void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg)
+int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg)
{
- /* For asynchronous message handling a valid receive-callback
- * is required.
- */
+ int rc = 0;
+
+ /* msg needs to have a valid receive-callback */
BUG_ON(!ap_msg->receive);
spin_lock_bh(&aq->lock);
- /* Queue the message. */
- list_add_tail(&ap_msg->list, &aq->requestq);
- aq->requestq_count++;
- aq->total_request_count++;
- atomic64_inc(&aq->card->total_request_count);
+
+ /* only allow to queue new messages if device state is ok */
+ if (aq->dev_state == AP_DEV_STATE_OPERATING) {
+ list_add_tail(&ap_msg->list, &aq->requestq);
+ aq->requestq_count++;
+ aq->total_request_count++;
+ atomic64_inc(&aq->card->total_request_count);
+ } else
+ rc = -ENODEV;
+
/* Send/receive as many request from the queue as possible. */
ap_wait(ap_sm_event_loop(aq, AP_SM_EVENT_POLL));
+
spin_unlock_bh(&aq->lock);
+
+ return rc;
}
EXPORT_SYMBOL(ap_queue_message);
@@ -749,8 +824,8 @@ void ap_queue_prepare_remove(struct ap_q
spin_lock_bh(&aq->lock);
/* flush queue */
__ap_flush_queue(aq);
- /* set REMOVE state to prevent new messages are queued in */
- aq->sm_state = AP_SM_STATE_REMOVE;
+ /* move queue device state to SHUTDOWN in progress */
+ aq->dev_state = AP_DEV_STATE_SHUTDOWN;
spin_unlock_bh(&aq->lock);
del_timer_sync(&aq->timeout);
}
@@ -758,21 +833,21 @@ void ap_queue_prepare_remove(struct ap_q
void ap_queue_remove(struct ap_queue *aq)
{
/*
- * all messages have been flushed and the state is
- * AP_SM_STATE_REMOVE. Now reset with zero which also
- * clears the irq registration and move the state
- * to AP_SM_STATE_UNBOUND to signal that this queue
- * is not used by any driver currently.
+ * all messages have been flushed and the device state
+ * is SHUTDOWN. Now reset with zero which also clears
+ * the irq registration and move the device state
+ * to the initial value AP_DEV_STATE_UNINITIATED.
*/
spin_lock_bh(&aq->lock);
ap_zapq(aq->qid);
- aq->sm_state = AP_SM_STATE_UNBOUND;
+ aq->dev_state = AP_DEV_STATE_UNINITIATED;
spin_unlock_bh(&aq->lock);
}
void ap_queue_init_state(struct ap_queue *aq)
{
spin_lock_bh(&aq->lock);
+ aq->dev_state = AP_DEV_STATE_OPERATING;
aq->sm_state = AP_SM_STATE_RESET_START;
ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL));
spin_unlock_bh(&aq->lock);
--- a/drivers/s390/crypto/zcrypt_msgtype50.c
+++ b/drivers/s390/crypto/zcrypt_msgtype50.c
@@ -471,7 +471,9 @@ static long zcrypt_cex2a_modexpo(struct
if (rc)
goto out_free;
init_completion(&work);
- ap_queue_message(zq->queue, &ap_msg);
+ rc = ap_queue_message(zq->queue, &ap_msg);
+ if (rc)
+ goto out_free;
rc = wait_for_completion_interruptible(&work);
if (rc == 0) {
rc = ap_msg.rc;
@@ -515,7 +517,9 @@ static long zcrypt_cex2a_modexpo_crt(str
if (rc)
goto out_free;
init_completion(&work);
- ap_queue_message(zq->queue, &ap_msg);
+ rc = ap_queue_message(zq->queue, &ap_msg);
+ if (rc)
+ goto out_free;
rc = wait_for_completion_interruptible(&work);
if (rc == 0) {
rc = ap_msg.rc;
--- a/drivers/s390/crypto/zcrypt_msgtype6.c
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -1027,7 +1027,9 @@ static long zcrypt_msgtype6_modexpo(stru
if (rc)
goto out_free;
init_completion(&resp_type.work);
- ap_queue_message(zq->queue, &ap_msg);
+ rc = ap_queue_message(zq->queue, &ap_msg);
+ if (rc)
+ goto out_free;
rc = wait_for_completion_interruptible(&resp_type.work);
if (rc == 0) {
rc = ap_msg.rc;
@@ -1071,7 +1073,9 @@ static long zcrypt_msgtype6_modexpo_crt(
if (rc)
goto out_free;
init_completion(&resp_type.work);
- ap_queue_message(zq->queue, &ap_msg);
+ rc = ap_queue_message(zq->queue, &ap_msg);
+ if (rc)
+ goto out_free;
rc = wait_for_completion_interruptible(&resp_type.work);
if (rc == 0) {
rc = ap_msg.rc;
@@ -1130,7 +1134,9 @@ static long zcrypt_msgtype6_send_cprb(bo
struct response_type *rtype = (struct response_type *)(ap_msg->private);
init_completion(&rtype->work);
- ap_queue_message(zq->queue, ap_msg);
+ rc = ap_queue_message(zq->queue, ap_msg);
+ if (rc)
+ goto out;
rc = wait_for_completion_interruptible(&rtype->work);
if (rc == 0) {
rc = ap_msg->rc;
@@ -1139,7 +1145,7 @@ static long zcrypt_msgtype6_send_cprb(bo
} else
/* Signal pending. */
ap_cancel_message(zq->queue, ap_msg);
-
+out:
return rc;
}
@@ -1232,7 +1238,9 @@ static long zcrypt_msgtype6_send_ep11_cp
}
init_completion(&rtype->work);
- ap_queue_message(zq->queue, ap_msg);
+ rc = ap_queue_message(zq->queue, ap_msg);
+ if (rc)
+ goto out;
rc = wait_for_completion_interruptible(&rtype->work);
if (rc == 0) {
rc = ap_msg->rc;
@@ -1241,7 +1249,7 @@ static long zcrypt_msgtype6_send_ep11_cp
} else
/* Signal pending. */
ap_cancel_message(zq->queue, ap_msg);
-
+out:
return rc;
}
@@ -1293,7 +1301,9 @@ static long zcrypt_msgtype6_rng(struct z
msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
init_completion(&rtype->work);
- ap_queue_message(zq->queue, ap_msg);
+ rc = ap_queue_message(zq->queue, ap_msg);
+ if (rc)
+ goto out;
rc = wait_for_completion_interruptible(&rtype->work);
if (rc == 0) {
rc = ap_msg->rc;
@@ -1302,7 +1312,7 @@ static long zcrypt_msgtype6_rng(struct z
} else
/* Signal pending. */
ap_cancel_message(zq->queue, ap_msg);
-
+out:
return rc;
}