From: Zhou Wang <wangzhou1@hisilicon.com>
Date: Sat, 9 May 2020 17:43:58 +0800
Subject: crypto: hisilicon/qm - add state machine for QM
Git-commit: b67202e8ed30bfa07b07a6f8fc762417a9a4e6de (partial)
Patch-mainline: v5.8-rc1
References: jsc#SLE-16507 jsc#SLE-15835
Add specific states for qm and qp, every state change under critical region
to prevent from race condition. Meanwhile, qp state change will also depend
on qm state.
Due to the introduction of these states, it is necessary to pay attention
to the calls of public logic, such as concurrent scenarios resetting and
releasing queue will call hisi_qm_stop, which needs to add additional
status to distinguish and process.
Signed-off-by: Zhou Wang <wangzhou1@hisilicon.com>
Signed-off-by: Shukun Tan <tanshukun1@huawei.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
[mb: drop hunk hisi_qm_get_available_instances and qm_set_sqctype as we don't support uacce]
Signed-off-by: Matthias Brugger <mbrugger@suse.com>
---
drivers/crypto/hisilicon/qm.c | 358 +++++++++++++++++++++++++++++++++---------
drivers/crypto/hisilicon/qm.h | 24 ++
2 files changed, 303 insertions(+), 79 deletions(-)
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -349,6 +349,93 @@ static const char * const qm_fifo_overfl
"cq", "eq", "aeq",
};
+static const char * const qm_s[] = {
+ "init", "start", "close", "stop",
+};
+
+static const char * const qp_s[] = {
+ "none", "init", "start", "stop", "close",
+};
+
+static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
+{
+ enum qm_state curr = atomic_read(&qm->status.flags);
+ bool avail = false;
+
+ switch (curr) {
+ case QM_INIT:
+ if (new == QM_START || new == QM_CLOSE)
+ avail = true;
+ break;
+ case QM_START:
+ if (new == QM_STOP)
+ avail = true;
+ break;
+ case QM_STOP:
+ if (new == QM_CLOSE || new == QM_START)
+ avail = true;
+ break;
+ default:
+ break;
+ }
+
+ dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
+ qm_s[curr], qm_s[new]);
+
+ if (!avail)
+ dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
+ qm_s[curr], qm_s[new]);
+
+ return avail;
+}
+
+static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
+ enum qp_state new)
+{
+ enum qm_state qm_curr = atomic_read(&qm->status.flags);
+ enum qp_state qp_curr = 0;
+ bool avail = false;
+
+ if (qp)
+ qp_curr = atomic_read(&qp->qp_status.flags);
+
+ switch (new) {
+ case QP_INIT:
+ if (qm_curr == QM_START || qm_curr == QM_INIT)
+ avail = true;
+ break;
+ case QP_START:
+ if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
+ (qm_curr == QM_START && qp_curr == QP_STOP))
+ avail = true;
+ break;
+ case QP_STOP:
+ if ((qm_curr == QM_START && qp_curr == QP_START) ||
+ (qp_curr == QP_INIT))
+ avail = true;
+ break;
+ case QP_CLOSE:
+ if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
+ (qm_curr == QM_START && qp_curr == QP_STOP) ||
+ (qm_curr == QM_STOP && qp_curr == QP_STOP) ||
+ (qm_curr == QM_STOP && qp_curr == QP_INIT))
+ avail = true;
+ break;
+ default:
+ break;
+ }
+
+ dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
+ qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
+
+ if (!avail)
+ dev_warn(&qm->pdev->dev,
+ "Can not change qp state from %s to %s in QM %s\n",
+ qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
+
+ return avail;
+}
+
/* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
static int qm_wait_mb_ready(struct hisi_qm *qm)
{
@@ -691,7 +778,7 @@ static void qm_init_qp_status(struct his
qp_status->sq_tail = 0;
qp_status->cq_head = 0;
qp_status->cqc_phase = true;
- qp_status->flags = 0;
+ atomic_set(&qp_status->flags, 0);
}
static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
@@ -1147,29 +1234,21 @@ static void *qm_get_avail_sqe(struct his
return qp->sqe + sq_tail * qp->qm->sqe_size;
}
-/**
- * hisi_qm_create_qp() - Create a queue pair from qm.
- * @qm: The qm we create a qp from.
- * @alg_type: Accelerator specific algorithm type in sqc.
- *
- * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
- * qp memory fails.
- */
-struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
+static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
{
struct device *dev = &qm->pdev->dev;
struct hisi_qp *qp;
int qp_id, ret;
+ if (!qm_qp_avail_state(qm, NULL, QP_INIT))
+ return ERR_PTR(-EPERM);
+
qp = kzalloc(sizeof(*qp), GFP_KERNEL);
if (!qp)
return ERR_PTR(-ENOMEM);
- write_lock(&qm->qps_lock);
-
qp_id = find_first_zero_bit(qm->qp_bitmap, qm->qp_num);
if (qp_id >= qm->qp_num) {
- write_unlock(&qm->qps_lock);
dev_info(&qm->pdev->dev, "QM all queues are busy!\n");
ret = -EBUSY;
goto err_free_qp;
@@ -1177,9 +1256,6 @@ struct hisi_qp *hisi_qm_create_qp(struct
set_bit(qp_id, qm->qp_bitmap);
qm->qp_array[qp_id] = qp;
qm->qp_in_used++;
-
- write_unlock(&qm->qps_lock);
-
qp->qm = qm;
if (qm->use_dma_api) {
@@ -1198,18 +1274,36 @@ struct hisi_qp *hisi_qm_create_qp(struct
qp->qp_id = qp_id;
qp->alg_type = alg_type;
+ atomic_set(&qp->qp_status.flags, QP_INIT);
return qp;
err_clear_bit:
- write_lock(&qm->qps_lock);
qm->qp_array[qp_id] = NULL;
clear_bit(qp_id, qm->qp_bitmap);
- write_unlock(&qm->qps_lock);
err_free_qp:
kfree(qp);
return ERR_PTR(ret);
}
+
+/**
+ * hisi_qm_create_qp() - Create a queue pair from qm.
+ * @qm: The qm we create a qp from.
+ * @alg_type: Accelerator specific algorithm type in sqc.
+ *
+ * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
+ * qp memory fails.
+ */
+struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
+{
+ struct hisi_qp *qp;
+
+ down_write(&qm->qps_lock);
+ qp = qm_create_qp_nolock(qm, alg_type);
+ up_write(&qm->qps_lock);
+
+ return qp;
+}
EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
/**
@@ -1224,16 +1318,23 @@ void hisi_qm_release_qp(struct hisi_qp *
struct qm_dma *qdma = &qp->qdma;
struct device *dev = &qm->pdev->dev;
+ down_write(&qm->qps_lock);
+
+ if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
+ up_write(&qm->qps_lock);
+ return;
+ }
+
if (qm->use_dma_api && qdma->va)
dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
- write_lock(&qm->qps_lock);
qm->qp_array[qp->qp_id] = NULL;
clear_bit(qp->qp_id, qm->qp_bitmap);
qm->qp_in_used--;
- write_unlock(&qm->qps_lock);
kfree(qp);
+
+ up_write(&qm->qps_lock);
}
EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
@@ -1304,15 +1405,7 @@ static int qm_qp_ctx_cfg(struct hisi_qp
return ret;
}
-/**
- * hisi_qm_start_qp() - Start a qp into running.
- * @qp: The qp we want to start to run.
- * @arg: Accelerator specific argument.
- *
- * After this function, qp can receive request from user. Return qp_id if
- * successful, Return -EBUSY if failed.
- */
-int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
+static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
{
struct hisi_qm *qm = qp->qm;
struct device *dev = &qm->pdev->dev;
@@ -1322,6 +1415,9 @@ int hisi_qm_start_qp(struct hisi_qp *qp,
size_t off = 0;
int ret;
+ if (!qm_qp_avail_state(qm, qp, QP_START))
+ return -EPERM;
+
#define QP_INIT_BUF(qp, type, size) do { \
(qp)->type = ((qp)->qdma.va + (off)); \
(qp)->type##_dma = (qp)->qdma.dma + (off); \
@@ -1352,10 +1448,31 @@ int hisi_qm_start_qp(struct hisi_qp *qp,
if (ret)
return ret;
+ atomic_set(&qp->qp_status.flags, QP_START);
dev_dbg(dev, "queue %d started\n", qp_id);
return qp_id;
}
+
+/**
+ * hisi_qm_start_qp() - Start a qp into running.
+ * @qp: The qp we want to start to run.
+ * @arg: Accelerator specific argument.
+ *
+ * After this function, qp can receive request from user. Return 0 if
+ * successful, Return -EBUSY if failed.
+ */
+int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
+{
+ struct hisi_qm *qm = qp->qm;
+ int ret;
+
+ down_write(&qm->qps_lock);
+ ret = qm_start_qp_nolock(qp, arg);
+ up_write(&qm->qps_lock);
+
+ return ret;
+}
EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
static void *qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
@@ -1459,20 +1576,26 @@ static int qm_drain_qp(struct hisi_qp *q
return ret;
}
-/**
- * hisi_qm_stop_qp() - Stop a qp in qm.
- * @qp: The qp we want to stop.
- *
- * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
- */
-int hisi_qm_stop_qp(struct hisi_qp *qp)
+static int qm_stop_qp_nolock(struct hisi_qp *qp)
{
struct device *dev = &qp->qm->pdev->dev;
int ret;
- /* it is stopped */
- if (test_bit(QP_STOP, &qp->qp_status.flags))
+ /*
+ * It is allowed to stop and release qp when reset, If the qp is
+ * stopped when reset but still want to be released then, the
+ * is_resetting flag should be set negative so that this qp will not
+ * be restarted after reset.
+ */
+ if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
+ qp->is_resetting = false;
return 0;
+ }
+
+ if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
+ return -EPERM;
+
+ atomic_set(&qp->qp_status.flags, QP_STOP);
ret = qm_drain_qp(qp);
if (ret)
@@ -1483,12 +1606,27 @@ int hisi_qm_stop_qp(struct hisi_qp *qp)
else
flush_work(&qp->qm->work);
- set_bit(QP_STOP, &qp->qp_status.flags);
-
dev_dbg(dev, "stop queue %u!", qp->qp_id);
return 0;
}
+
+/**
+ * hisi_qm_stop_qp() - Stop a qp in qm.
+ * @qp: The qp we want to stop.
+ *
+ * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
+ */
+int hisi_qm_stop_qp(struct hisi_qp *qp)
+{
+ int ret;
+
+ down_write(&qp->qm->qps_lock);
+ ret = qm_stop_qp_nolock(qp);
+ up_write(&qp->qm->qps_lock);
+
+ return ret;
+}
EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
/**
@@ -1498,6 +1636,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
*
* This function will return -EBUSY if qp is currently full, and -EAGAIN
* if qp related qm is resetting.
+ *
+ * Note: This function may run with qm_irq_thread and ACC reset at same time.
+ * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
+ * reset may happen, we have no lock here considering performance. This
+ * causes current qm_db sending fail or can not receive sended sqe. QM
+ * sync/async receive function should handle the error sqe. ACC reset
+ * done function should clear used sqe to 0.
*/
int hisi_qp_send(struct hisi_qp *qp, const void *msg)
{
@@ -1506,7 +1651,9 @@ int hisi_qp_send(struct hisi_qp *qp, con
u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
void *sqe = qm_get_avail_sqe(qp);
- if (unlikely(test_bit(QP_STOP, &qp->qp_status.flags))) {
+ if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
+ atomic_read(&qp->qm->status.flags) == QM_STOP ||
+ qp->is_resetting)) {
dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
return -EAGAIN;
}
@@ -1546,9 +1693,9 @@ int hisi_qm_get_free_qp_num(struct hisi_
{
int ret;
- read_lock(&qm->qps_lock);
+ down_read(&qm->qps_lock);
ret = qm->qp_num - qm->qp_in_used;
- read_unlock(&qm->qps_lock);
+ up_read(&qm->qps_lock);
return ret;
}
@@ -1619,9 +1766,10 @@ int hisi_qm_init(struct hisi_qm *qm)
qm->qp_in_used = 0;
mutex_init(&qm->mailbox_lock);
- rwlock_init(&qm->qps_lock);
+ init_rwsem(&qm->qps_lock);
INIT_WORK(&qm->work, qm_work_process);
+ atomic_set(&qm->status.flags, QM_INIT);
dev_dbg(dev, "init qm %s with %s\n", pdev->is_physfn ? "pf" : "vf",
qm->use_dma_api ? "dma api" : "iommu api");
@@ -1651,6 +1799,13 @@ void hisi_qm_uninit(struct hisi_qm *qm)
struct pci_dev *pdev = qm->pdev;
struct device *dev = &pdev->dev;
+ down_write(&qm->qps_lock);
+
+ if (!qm_avail_state(qm, QM_CLOSE)) {
+ up_write(&qm->qps_lock);
+ return;
+ }
+
if (qm->use_dma_api && qm->qdma.va) {
hisi_qm_cache_wb(qm);
dma_free_coherent(dev, qm->qdma.size,
@@ -1663,6 +1818,8 @@ void hisi_qm_uninit(struct hisi_qm *qm)
iounmap(qm->io_base);
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
+
+ up_write(&qm->qps_lock);
}
EXPORT_SYMBOL_GPL(hisi_qm_uninit);
@@ -1845,12 +2002,21 @@ static int __hisi_qm_start(struct hisi_q
int hisi_qm_start(struct hisi_qm *qm)
{
struct device *dev = &qm->pdev->dev;
+ int ret = 0;
+
+ down_write(&qm->qps_lock);
+
+ if (!qm_avail_state(qm, QM_START)) {
+ up_write(&qm->qps_lock);
+ return -EPERM;
+ }
dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num);
if (!qm->qp_num) {
dev_err(dev, "qp_num should not be 0\n");
- return -EINVAL;
+ ret = -EINVAL;
+ goto err_unlock;
}
if (!qm->qp_bitmap) {
@@ -1859,12 +2025,15 @@ int hisi_qm_start(struct hisi_qm *qm)
qm->qp_array = devm_kcalloc(dev, qm->qp_num,
sizeof(struct hisi_qp *),
GFP_KERNEL);
- if (!qm->qp_bitmap || !qm->qp_array)
- return -ENOMEM;
+ if (!qm->qp_bitmap || !qm->qp_array) {
+ ret = -ENOMEM;
+ goto err_unlock;
+ }
}
if (!qm->use_dma_api) {
dev_dbg(&qm->pdev->dev, "qm delay start\n");
+ up_write(&qm->qps_lock);
return 0;
} else if (!qm->qdma.va) {
qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) +
@@ -1875,11 +2044,19 @@ int hisi_qm_start(struct hisi_qm *qm)
&qm->qdma.dma, GFP_KERNEL);
dev_dbg(dev, "allocate qm dma buf(va=%pK, dma=%pad, size=%zx)\n",
qm->qdma.va, &qm->qdma.dma, qm->qdma.size);
- if (!qm->qdma.va)
- return -ENOMEM;
+ if (!qm->qdma.va) {
+ ret = -ENOMEM;
+ goto err_unlock;
+ }
}
- return __hisi_qm_start(qm);
+ ret = __hisi_qm_start(qm);
+ if (!ret)
+ atomic_set(&qm->status.flags, QM_START);
+
+err_unlock:
+ up_write(&qm->qps_lock);
+ return ret;
}
EXPORT_SYMBOL_GPL(hisi_qm_start);
@@ -1893,20 +2070,44 @@ static int qm_restart(struct hisi_qm *qm
if (ret < 0)
return ret;
- write_lock(&qm->qps_lock);
+ down_write(&qm->qps_lock);
for (i = 0; i < qm->qp_num; i++) {
qp = qm->qp_array[i];
- if (qp) {
- ret = hisi_qm_start_qp(qp, 0);
+ if (qp && atomic_read(&qp->qp_status.flags) == QP_STOP &&
+ qp->is_resetting == true) {
+ ret = qm_start_qp_nolock(qp, 0);
if (ret < 0) {
dev_err(dev, "Failed to start qp%d!\n", i);
- write_unlock(&qm->qps_lock);
+ up_write(&qm->qps_lock);
+ return ret;
+ }
+ qp->is_resetting = false;
+ }
+ }
+ up_write(&qm->qps_lock);
+
+ return 0;
+}
+
+/* Stop started qps in reset flow */
+static int qm_stop_started_qp(struct hisi_qm *qm)
+{
+ struct device *dev = &qm->pdev->dev;
+ struct hisi_qp *qp;
+ int i, ret;
+
+ for (i = 0; i < qm->qp_num; i++) {
+ qp = qm->qp_array[i];
+ if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
+ qp->is_resetting = true;
+ ret = qm_stop_qp_nolock(qp);
+ if (ret < 0) {
+ dev_err(dev, "Failed to stop qp%d!\n", i);
return ret;
}
}
}
- write_unlock(&qm->qps_lock);
return 0;
}
@@ -1922,7 +2123,7 @@ static void qm_clear_queues(struct hisi_
for (i = 0; i < qm->qp_num; i++) {
qp = qm->qp_array[i];
- if (qp)
+ if (qp && qp->is_resetting)
memset(qp->qdma.va, 0, qp->qdma.size);
}
@@ -1939,41 +2140,43 @@ static void qm_clear_queues(struct hisi_
*/
int hisi_qm_stop(struct hisi_qm *qm)
{
- struct device *dev;
- struct hisi_qp *qp;
- int ret = 0, i;
+ struct device *dev = &qm->pdev->dev;
+ int ret = 0;
- if (!qm || !qm->pdev) {
- WARN_ON(1);
- return -EINVAL;
+ down_write(&qm->qps_lock);
+
+ if (!qm_avail_state(qm, QM_STOP)) {
+ ret = -EPERM;
+ goto err_unlock;
}
- dev = &qm->pdev->dev;
+ if (qm->status.stop_reason == QM_SOFT_RESET ||
+ qm->status.stop_reason == QM_FLR) {
+ ret = qm_stop_started_qp(qm);
+ if (ret < 0) {
+ dev_err(dev, "Failed to stop started qp!\n");
+ goto err_unlock;
+ }
+ }
/* Mask eq and aeq irq */
writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
- /* Stop all qps belong to this qm */
- for (i = 0; i < qm->qp_num; i++) {
- qp = qm->qp_array[i];
- if (qp) {
- ret = hisi_qm_stop_qp(qp);
- if (ret < 0) {
- dev_err(dev, "Failed to stop qp%d!\n", i);
- return -EBUSY;
- }
- }
- }
-
if (qm->fun_type == QM_HW_PF) {
ret = hisi_qm_set_vft(qm, 0, 0, 0);
- if (ret < 0)
+ if (ret < 0) {
dev_err(dev, "Failed to set vft!\n");
+ ret = -EBUSY;
+ goto err_unlock;
+ }
}
qm_clear_queues(qm);
+ atomic_set(&qm->status.flags, QM_STOP);
+err_unlock:
+ up_write(&qm->qps_lock);
return ret;
}
EXPORT_SYMBOL_GPL(hisi_qm_stop);
@@ -2545,6 +2748,7 @@ static int qm_set_msi(struct hisi_qm *qm
static int qm_vf_reset_prepare(struct hisi_qm *qm)
{
struct hisi_qm_list *qm_list = qm->qm_list;
+ int stop_reason = qm->status.stop_reason;
struct pci_dev *pdev = qm->pdev;
struct pci_dev *virtfn;
struct hisi_qm *vf_qm;
@@ -2557,6 +2761,7 @@ static int qm_vf_reset_prepare(struct hi
continue;
if (pci_physfn(virtfn) == pdev) {
+ vf_qm->status.stop_reason = stop_reason;
ret = hisi_qm_stop(vf_qm);
if (ret)
goto stop_fail;
@@ -2603,6 +2808,7 @@ static int qm_controller_reset_prepare(s
}
}
+ qm->status.stop_reason = QM_SOFT_RESET;
ret = hisi_qm_stop(qm);
if (ret) {
pci_err(pdev, "Fails to stop QM!\n");
--- a/drivers/crypto/hisilicon/qm.h
+++ b/drivers/crypto/hisilicon/qm.h
@@ -81,8 +81,24 @@
#define HISI_ACC_SGL_SGE_NR_MAX 255
+enum qm_stop_reason {
+ QM_NORMAL,
+ QM_SOFT_RESET,
+ QM_FLR,
+};
+
+enum qm_state {
+ QM_INIT = 0,
+ QM_START,
+ QM_CLOSE,
+ QM_STOP,
+};
+
enum qp_state {
+ QP_INIT = 1,
+ QP_START,
QP_STOP,
+ QP_CLOSE,
};
enum qm_hw_ver {
@@ -126,7 +142,8 @@ struct hisi_qm_status {
bool eqc_phase;
u32 aeq_head;
bool aeqc_phase;
- unsigned long flags;
+ atomic_t flags;
+ int stop_reason;
};
struct hisi_qm;
@@ -193,7 +210,7 @@ struct hisi_qm {
struct hisi_qm_err_status err_status;
unsigned long reset_flag;
- rwlock_t qps_lock;
+ struct rw_semaphore qps_lock;
unsigned long *qp_bitmap;
struct hisi_qp **qp_array;
@@ -217,7 +234,7 @@ struct hisi_qp_status {
u16 sq_tail;
u16 cq_head;
bool cqc_phase;
- unsigned long flags;
+ atomic_t flags;
};
struct hisi_qp_ops {
@@ -241,6 +258,7 @@ struct hisi_qp {
void (*req_cb)(struct hisi_qp *qp, void *data);
struct hisi_qm *qm;
+ bool is_resetting;
};
static inline int q_num_set(const char *val, const struct kernel_param *kp,