From: Leon Romanovsky <leonro@nvidia.com>
Date: Wed, 6 Apr 2022 11:25:49 +0300
Subject: net/mlx5: Move IPsec file to relevant directory
Patch-mainline: v5.19-rc1
Git-commit: 16fe5a1c5c074a836626e3bd9560d3c4a39a3fcf
References: jsc#PED-1549
IPsec is part of ethernet side of mlx5 driver and needs to be placed
in en_accel folder.
Link: https://lore.kernel.org/r/a0ca88f4d9c602c574106c0de0511803e7dcbdff.1649232994.git.leonro@nvidia.com
Reviewed-by: Raed Salem <raeds@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +-
drivers/net/ethernet/mellanox/mlx5/core/en/params.c | 2 +-
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h | 2 +-
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c | 2 +-
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h | 2 +-
.../mellanox/mlx5/core/{accel => en_accel}/ipsec_offload.c | 2 +-
.../mellanox/mlx5/core/{accel => en_accel}/ipsec_offload.h | 0
drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2
drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c | 524 ----------
drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.h | 49
drivers/net/ethernet/mellanox/mlx5/core/en/params.c | 2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h | 2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c | 2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h | 2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c | 524 ++++++++++
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.h | 49
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c | 2
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c | 2
drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 2
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 2
drivers/net/ethernet/mellanox/mlx5/core/main.c | 2
14 files changed, 583 insertions(+), 583 deletions(-)
rename drivers/net/ethernet/mellanox/mlx5/core/{accel => en_accel}/ipsec_offload.c (99%)
rename drivers/net/ethernet/mellanox/mlx5/core/{accel => en_accel}/ipsec_offload.h (100%)
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -94,7 +94,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cm
mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
en_accel/ipsec_stats.o en_accel/ipsec_fs.o \
- accel/ipsec_offload.o
+ en_accel/ipsec_offload.o
mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/ktls_stats.o \
en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \
--- a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c
+++ /dev/null
@@ -1,524 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
-/* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */
-
-#include "mlx5_core.h"
-#include "accel/ipsec_offload.h"
-#include "lib/mlx5.h"
-#include "en_accel/ipsec_fs.h"
-
-struct mlx5_ipsec_sa_ctx {
- struct rhash_head hash;
- u32 enc_key_id;
- u32 ipsec_obj_id;
- /* hw ctx */
- struct mlx5_core_dev *dev;
- struct mlx5_ipsec_esp_xfrm *mxfrm;
-};
-
-struct mlx5_ipsec_esp_xfrm {
- /* reference counter of SA ctx */
- struct mlx5_ipsec_sa_ctx *sa_ctx;
- struct mutex lock; /* protects mlx5_ipsec_esp_xfrm */
- struct mlx5_accel_esp_xfrm accel_xfrm;
-};
-
-u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
-{
- u32 caps;
-
- if (!MLX5_CAP_GEN(mdev, ipsec_offload))
- return 0;
-
- if (!MLX5_CAP_GEN(mdev, log_max_dek))
- return 0;
-
- if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
- MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
- return 0;
-
- if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_offload) ||
- !MLX5_CAP_ETH(mdev, insert_trailer))
- return 0;
-
- if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
- !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
- return 0;
-
- caps = MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 |
- MLX5_ACCEL_IPSEC_CAP_LSO;
-
- if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) &&
- MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
- caps |= MLX5_ACCEL_IPSEC_CAP_ESP;
-
- if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
- caps |= MLX5_ACCEL_IPSEC_CAP_ESN;
- caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
- }
-
- /* We can accommodate up to 2^24 different IPsec objects
- * because we use up to 24 bit in flow table metadata
- * to hold the IPsec Object unique handle.
- */
- WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
- return caps;
-}
-EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
-
-static int
-mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
- const struct mlx5_accel_esp_xfrm_attrs *attrs)
-{
- if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
- mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n",
- attrs->replay_type);
- return -EOPNOTSUPP;
- }
-
- if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
- mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n",
- attrs->keymat_type);
- return -EOPNOTSUPP;
- }
-
- if (attrs->keymat.aes_gcm.iv_algo !=
- MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
- mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n",
- attrs->keymat.aes_gcm.iv_algo);
- return -EOPNOTSUPP;
- }
-
- if (attrs->keymat.aes_gcm.key_len != 128 &&
- attrs->keymat.aes_gcm.key_len != 256) {
- mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n",
- attrs->keymat.aes_gcm.key_len);
- return -EOPNOTSUPP;
- }
-
- if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
- !MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
- mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n");
- return -EOPNOTSUPP;
- }
-
- return 0;
-}
-
-static struct mlx5_accel_esp_xfrm *
-mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev,
- const struct mlx5_accel_esp_xfrm_attrs *attrs,
- u32 flags)
-{
- struct mlx5_ipsec_esp_xfrm *mxfrm;
- int err = 0;
-
- err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs);
- if (err)
- return ERR_PTR(err);
-
- mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL);
- if (!mxfrm)
- return ERR_PTR(-ENOMEM);
-
- mutex_init(&mxfrm->lock);
- memcpy(&mxfrm->accel_xfrm.attrs, attrs,
- sizeof(mxfrm->accel_xfrm.attrs));
-
- return &mxfrm->accel_xfrm;
-}
-
-static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
-{
- struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm,
- accel_xfrm);
-
- /* assuming no sa_ctx are connected to this xfrm_ctx */
- WARN_ON(mxfrm->sa_ctx);
- kfree(mxfrm);
-}
-
-struct mlx5_ipsec_obj_attrs {
- const struct aes_gcm_keymat *aes_gcm;
- u32 accel_flags;
- u32 esn_msb;
- u32 enc_key_id;
-};
-
-static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev,
- struct mlx5_ipsec_obj_attrs *attrs,
- u32 *ipsec_id)
-{
- const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm;
- u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
- u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
- void *obj, *salt_p, *salt_iv_p;
- int err;
-
- obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
-
- /* salt and seq_iv */
- salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
- memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
-
- switch (aes_gcm->icv_len) {
- case 64:
- MLX5_SET(ipsec_obj, obj, icv_length,
- MLX5_IPSEC_OBJECT_ICV_LEN_8B);
- break;
- case 96:
- MLX5_SET(ipsec_obj, obj, icv_length,
- MLX5_IPSEC_OBJECT_ICV_LEN_12B);
- break;
- case 128:
- MLX5_SET(ipsec_obj, obj, icv_length,
- MLX5_IPSEC_OBJECT_ICV_LEN_16B);
- break;
- default:
- return -EINVAL;
- }
- salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
- memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
- /* esn */
- if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
- MLX5_SET(ipsec_obj, obj, esn_en, 1);
- MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
- if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
- MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
- }
-
- MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id);
-
- /* general object fields set */
- MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
- MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
- MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
- MLX5_GENERAL_OBJECT_TYPES_IPSEC);
-
- err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
- if (!err)
- *ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
-
- return err;
-}
-
-static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id)
-{
- u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
- u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
-
- MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
- MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
- MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
- MLX5_GENERAL_OBJECT_TYPES_IPSEC);
- MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
-
- mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-}
-
-static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev,
- struct mlx5_accel_esp_xfrm *accel_xfrm,
- const __be32 saddr[4], const __be32 daddr[4],
- const __be32 spi, bool is_ipv6, u32 *hw_handle)
-{
- struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs;
- struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
- struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
- struct mlx5_ipsec_esp_xfrm *mxfrm;
- struct mlx5_ipsec_sa_ctx *sa_ctx;
- int err;
-
- /* alloc SA context */
- sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
- if (!sa_ctx)
- return ERR_PTR(-ENOMEM);
-
- sa_ctx->dev = mdev;
-
- mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
- mutex_lock(&mxfrm->lock);
- sa_ctx->mxfrm = mxfrm;
-
- /* key */
- err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
- aes_gcm->key_len / BITS_PER_BYTE,
- MLX5_ACCEL_OBJ_IPSEC_KEY,
- &sa_ctx->enc_key_id);
- if (err) {
- mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
- goto err_sa_ctx;
- }
-
- ipsec_attrs.aes_gcm = aes_gcm;
- ipsec_attrs.accel_flags = accel_xfrm->attrs.flags;
- ipsec_attrs.esn_msb = accel_xfrm->attrs.esn;
- ipsec_attrs.enc_key_id = sa_ctx->enc_key_id;
- err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs,
- &sa_ctx->ipsec_obj_id);
- if (err) {
- mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
- goto err_enc_key;
- }
-
- *hw_handle = sa_ctx->ipsec_obj_id;
- mxfrm->sa_ctx = sa_ctx;
- mutex_unlock(&mxfrm->lock);
-
- return sa_ctx;
-
-err_enc_key:
- mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id);
-err_sa_ctx:
- mutex_unlock(&mxfrm->lock);
- kfree(sa_ctx);
- return ERR_PTR(err);
-}
-
-static void mlx5_ipsec_offload_delete_sa_ctx(void *context)
-{
- struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context;
- struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm;
-
- mutex_lock(&mxfrm->lock);
- mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id);
- mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id);
- kfree(sa_ctx);
- mxfrm->sa_ctx = NULL;
- mutex_unlock(&mxfrm->lock);
-}
-
-static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev)
-{
- return 0;
-}
-
-static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev,
- struct mlx5_ipsec_obj_attrs *attrs,
- u32 ipsec_id)
-{
- u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
- u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
- u64 modify_field_select = 0;
- u64 general_obj_types;
- void *obj;
- int err;
-
- if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED))
- return 0;
-
- general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
- if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
- return -EINVAL;
-
- /* general object fields set */
- MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
- MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
- MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
- err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
- if (err) {
- mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
- ipsec_id, err);
- return err;
- }
-
- obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
- modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
-
- /* esn */
- if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
- !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
- return -EOPNOTSUPP;
-
- obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
- MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
- if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
- MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
-
- /* general object fields set */
- MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
-
- return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-}
-
-static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
- const struct mlx5_accel_esp_xfrm_attrs *attrs)
-{
- struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
- struct mlx5_core_dev *mdev = xfrm->mdev;
- struct mlx5_ipsec_esp_xfrm *mxfrm;
-
- int err = 0;
-
- if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
- return 0;
-
- if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs))
- return -EOPNOTSUPP;
-
- mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
-
- mutex_lock(&mxfrm->lock);
-
- if (!mxfrm->sa_ctx)
- /* Not bound xfrm, change only sw attrs */
- goto change_sw_xfrm_attrs;
-
- /* need to add find and replace in ipsec_rhash_sa the sa_ctx */
- /* modify device with new hw_sa */
- ipsec_attrs.accel_flags = attrs->flags;
- ipsec_attrs.esn_msb = attrs->esn;
- err = mlx5_modify_ipsec_obj(mdev,
- &ipsec_attrs,
- mxfrm->sa_ctx->ipsec_obj_id);
-
-change_sw_xfrm_attrs:
- if (!err)
- memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));
-
- mutex_unlock(&mxfrm->lock);
- return err;
-}
-
-static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = {
- .create_hw_context = mlx5_ipsec_offload_create_sa_ctx,
- .free_hw_context = mlx5_ipsec_offload_delete_sa_ctx,
- .init = mlx5_ipsec_offload_init,
- .esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm,
- .esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm,
- .esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm,
-};
-
-static const struct mlx5_accel_ipsec_ops *
-mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev)
-{
- if (!mlx5_ipsec_device_caps(mdev))
- return NULL;
-
- return &ipsec_offload_ops;
-}
-
-void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops;
- int err = 0;
-
- ipsec_ops = mlx5_ipsec_offload_ops(mdev);
- if (!ipsec_ops || !ipsec_ops->init) {
- mlx5_core_dbg(mdev, "IPsec ops is not supported\n");
- return;
- }
-
- err = ipsec_ops->init(mdev);
- if (err) {
- mlx5_core_warn_once(
- mdev, "Failed to start IPsec device, err = %d\n", err);
- return;
- }
-
- mdev->ipsec_ops = ipsec_ops;
-}
-
-void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->cleanup)
- return;
-
- ipsec_ops->cleanup(mdev);
-}
-
-unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->counters_count)
- return -EOPNOTSUPP;
-
- return ipsec_ops->counters_count(mdev);
-}
-
-int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
- unsigned int count)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->counters_read)
- return -EOPNOTSUPP;
-
- return ipsec_ops->counters_read(mdev, counters, count);
-}
-
-void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,
- struct mlx5_accel_esp_xfrm *xfrm,
- u32 *sa_handle)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
- __be32 saddr[4] = {}, daddr[4] = {};
-
- if (!ipsec_ops || !ipsec_ops->create_hw_context)
- return ERR_PTR(-EOPNOTSUPP);
-
- if (!xfrm->attrs.is_ipv6) {
- saddr[3] = xfrm->attrs.saddr.a4;
- daddr[3] = xfrm->attrs.daddr.a4;
- } else {
- memcpy(saddr, xfrm->attrs.saddr.a6, sizeof(saddr));
- memcpy(daddr, xfrm->attrs.daddr.a6, sizeof(daddr));
- }
-
- return ipsec_ops->create_hw_context(mdev, xfrm, saddr, daddr,
- xfrm->attrs.spi,
- xfrm->attrs.is_ipv6, sa_handle);
-}
-
-void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->free_hw_context)
- return;
-
- ipsec_ops->free_hw_context(context);
-}
-
-struct mlx5_accel_esp_xfrm *
-mlx5_accel_esp_create_xfrm(struct mlx5_core_dev *mdev,
- const struct mlx5_accel_esp_xfrm_attrs *attrs)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
- struct mlx5_accel_esp_xfrm *xfrm;
-
- if (!ipsec_ops || !ipsec_ops->esp_create_xfrm)
- return ERR_PTR(-EOPNOTSUPP);
-
- xfrm = ipsec_ops->esp_create_xfrm(mdev, attrs, 0);
- if (IS_ERR(xfrm))
- return xfrm;
-
- xfrm->mdev = mdev;
- return xfrm;
-}
-EXPORT_SYMBOL_GPL(mlx5_accel_esp_create_xfrm);
-
-void mlx5_accel_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->esp_destroy_xfrm)
- return;
-
- ipsec_ops->esp_destroy_xfrm(xfrm);
-}
-EXPORT_SYMBOL_GPL(mlx5_accel_esp_destroy_xfrm);
-
-int mlx5_accel_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
- const struct mlx5_accel_esp_xfrm_attrs *attrs)
-{
- const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops;
-
- if (!ipsec_ops || !ipsec_ops->esp_modify_xfrm)
- return -EOPNOTSUPP;
-
- return ipsec_ops->esp_modify_xfrm(xfrm, attrs);
-}
-EXPORT_SYMBOL_GPL(mlx5_accel_esp_modify_xfrm);
--- a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
-/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
-
-#ifndef __MLX5_IPSEC_OFFLOAD_H__
-#define __MLX5_IPSEC_OFFLOAD_H__
-
-#include <linux/mlx5/driver.h>
-#include <linux/mlx5/accel.h>
-
-#ifdef CONFIG_MLX5_EN_IPSEC
-
-unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);
-int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
- unsigned int count);
-
-void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,
- struct mlx5_accel_esp_xfrm *xfrm,
- u32 *sa_handle);
-void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context);
-
-void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);
-void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
-
-struct mlx5_accel_ipsec_ops {
- unsigned int (*counters_count)(struct mlx5_core_dev *mdev);
- int (*counters_read)(struct mlx5_core_dev *mdev, u64 *counters,
- unsigned int count);
- void *(*create_hw_context)(struct mlx5_core_dev *mdev,
- struct mlx5_accel_esp_xfrm *xfrm,
- const __be32 saddr[4], const __be32 daddr[4],
- const __be32 spi, bool is_ipv6,
- u32 *sa_handle);
- void (*free_hw_context)(void *context);
- int (*init)(struct mlx5_core_dev *mdev);
- void (*cleanup)(struct mlx5_core_dev *mdev);
- struct mlx5_accel_esp_xfrm *(*esp_create_xfrm)(
- struct mlx5_core_dev *mdev,
- const struct mlx5_accel_esp_xfrm_attrs *attrs, u32 flags);
- int (*esp_modify_xfrm)(struct mlx5_accel_esp_xfrm *xfrm,
- const struct mlx5_accel_esp_xfrm_attrs *attrs);
- void (*esp_destroy_xfrm)(struct mlx5_accel_esp_xfrm *xfrm);
-};
-
-#else
-static inline void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) {}
-
-static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) {}
-#endif /* CONFIG_MLX5_EN_IPSEC */
-#endif /* __MLX5_IPSEC_OFFLOAD_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -5,7 +5,7 @@
#include "en/txrx.h"
#include "en/port.h"
#include "en_accel/en_accel.h"
-#include "accel/ipsec_offload.h"
+#include "en_accel/ipsec_offload.h"
static bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -40,7 +40,7 @@
#include <net/xfrm.h>
#include <linux/idr.h>
-#include "accel/ipsec_offload.h"
+#include "ipsec_offload.h"
#define MLX5E_IPSEC_SADB_RX_BITS 10
#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
@@ -2,7 +2,7 @@
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
#include <linux/netdevice.h>
-#include "accel/ipsec_offload.h"
+#include "ipsec_offload.h"
#include "ipsec_fs.h"
#include "fs_core.h"
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h
@@ -6,7 +6,7 @@
#include "en.h"
#include "ipsec.h"
-#include "accel/ipsec_offload.h"
+#include "ipsec_offload.h"
#include "en/fs.h"
#ifdef CONFIG_MLX5_EN_IPSEC
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
@@ -0,0 +1,524 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */
+
+#include "mlx5_core.h"
+#include "ipsec_offload.h"
+#include "lib/mlx5.h"
+#include "en_accel/ipsec_fs.h"
+
+struct mlx5_ipsec_sa_ctx {
+ struct rhash_head hash;
+ u32 enc_key_id;
+ u32 ipsec_obj_id;
+ /* hw ctx */
+ struct mlx5_core_dev *dev;
+ struct mlx5_ipsec_esp_xfrm *mxfrm;
+};
+
+struct mlx5_ipsec_esp_xfrm {
+ /* reference counter of SA ctx */
+ struct mlx5_ipsec_sa_ctx *sa_ctx;
+ struct mutex lock; /* protects mlx5_ipsec_esp_xfrm */
+ struct mlx5_accel_esp_xfrm accel_xfrm;
+};
+
+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
+{
+ u32 caps;
+
+ if (!MLX5_CAP_GEN(mdev, ipsec_offload))
+ return 0;
+
+ if (!MLX5_CAP_GEN(mdev, log_max_dek))
+ return 0;
+
+ if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+ return 0;
+
+ if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_offload) ||
+ !MLX5_CAP_ETH(mdev, insert_trailer))
+ return 0;
+
+ if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
+ !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
+ return 0;
+
+ caps = MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 |
+ MLX5_ACCEL_IPSEC_CAP_LSO;
+
+ if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) &&
+ MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
+ caps |= MLX5_ACCEL_IPSEC_CAP_ESP;
+
+ if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
+ caps |= MLX5_ACCEL_IPSEC_CAP_ESN;
+ caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
+ }
+
+ /* We can accommodate up to 2^24 different IPsec objects
+ * because we use up to 24 bit in flow table metadata
+ * to hold the IPsec Object unique handle.
+ */
+ WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
+ return caps;
+}
+EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
+
+static int
+mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
+ mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n",
+ attrs->replay_type);
+ return -EOPNOTSUPP;
+ }
+
+ if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
+ mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n",
+ attrs->keymat_type);
+ return -EOPNOTSUPP;
+ }
+
+ if (attrs->keymat.aes_gcm.iv_algo !=
+ MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
+ mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n",
+ attrs->keymat.aes_gcm.iv_algo);
+ return -EOPNOTSUPP;
+ }
+
+ if (attrs->keymat.aes_gcm.key_len != 128 &&
+ attrs->keymat.aes_gcm.key_len != 256) {
+ mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n",
+ attrs->keymat.aes_gcm.key_len);
+ return -EOPNOTSUPP;
+ }
+
+ if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
+ !MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
+ mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n");
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static struct mlx5_accel_esp_xfrm *
+mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs,
+ u32 flags)
+{
+ struct mlx5_ipsec_esp_xfrm *mxfrm;
+ int err = 0;
+
+ err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs);
+ if (err)
+ return ERR_PTR(err);
+
+ mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL);
+ if (!mxfrm)
+ return ERR_PTR(-ENOMEM);
+
+ mutex_init(&mxfrm->lock);
+ memcpy(&mxfrm->accel_xfrm.attrs, attrs,
+ sizeof(mxfrm->accel_xfrm.attrs));
+
+ return &mxfrm->accel_xfrm;
+}
+
+static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
+{
+ struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm,
+ accel_xfrm);
+
+ /* assuming no sa_ctx are connected to this xfrm_ctx */
+ WARN_ON(mxfrm->sa_ctx);
+ kfree(mxfrm);
+}
+
+struct mlx5_ipsec_obj_attrs {
+ const struct aes_gcm_keymat *aes_gcm;
+ u32 accel_flags;
+ u32 esn_msb;
+ u32 enc_key_id;
+};
+
+static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev,
+ struct mlx5_ipsec_obj_attrs *attrs,
+ u32 *ipsec_id)
+{
+ const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm;
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+ u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
+ void *obj, *salt_p, *salt_iv_p;
+ int err;
+
+ obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
+
+ /* salt and seq_iv */
+ salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
+ memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
+
+ switch (aes_gcm->icv_len) {
+ case 64:
+ MLX5_SET(ipsec_obj, obj, icv_length,
+ MLX5_IPSEC_OBJECT_ICV_LEN_8B);
+ break;
+ case 96:
+ MLX5_SET(ipsec_obj, obj, icv_length,
+ MLX5_IPSEC_OBJECT_ICV_LEN_12B);
+ break;
+ case 128:
+ MLX5_SET(ipsec_obj, obj, icv_length,
+ MLX5_IPSEC_OBJECT_ICV_LEN_16B);
+ break;
+ default:
+ return -EINVAL;
+ }
+ salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
+ memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
+ /* esn */
+ if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
+ MLX5_SET(ipsec_obj, obj, esn_en, 1);
+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
+ if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
+ MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
+ }
+
+ MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id);
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+ MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (!err)
+ *ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+ return err;
+}
+
+static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id)
+{
+ u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+ MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
+
+ mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev,
+ struct mlx5_accel_esp_xfrm *accel_xfrm,
+ const __be32 saddr[4], const __be32 daddr[4],
+ const __be32 spi, bool is_ipv6, u32 *hw_handle)
+{
+ struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs;
+ struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
+ struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
+ struct mlx5_ipsec_esp_xfrm *mxfrm;
+ struct mlx5_ipsec_sa_ctx *sa_ctx;
+ int err;
+
+ /* alloc SA context */
+ sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
+ if (!sa_ctx)
+ return ERR_PTR(-ENOMEM);
+
+ sa_ctx->dev = mdev;
+
+ mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
+ mutex_lock(&mxfrm->lock);
+ sa_ctx->mxfrm = mxfrm;
+
+ /* key */
+ err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
+ aes_gcm->key_len / BITS_PER_BYTE,
+ MLX5_ACCEL_OBJ_IPSEC_KEY,
+ &sa_ctx->enc_key_id);
+ if (err) {
+ mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
+ goto err_sa_ctx;
+ }
+
+ ipsec_attrs.aes_gcm = aes_gcm;
+ ipsec_attrs.accel_flags = accel_xfrm->attrs.flags;
+ ipsec_attrs.esn_msb = accel_xfrm->attrs.esn;
+ ipsec_attrs.enc_key_id = sa_ctx->enc_key_id;
+ err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs,
+ &sa_ctx->ipsec_obj_id);
+ if (err) {
+ mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
+ goto err_enc_key;
+ }
+
+ *hw_handle = sa_ctx->ipsec_obj_id;
+ mxfrm->sa_ctx = sa_ctx;
+ mutex_unlock(&mxfrm->lock);
+
+ return sa_ctx;
+
+err_enc_key:
+ mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id);
+err_sa_ctx:
+ mutex_unlock(&mxfrm->lock);
+ kfree(sa_ctx);
+ return ERR_PTR(err);
+}
+
+static void mlx5_ipsec_offload_delete_sa_ctx(void *context)
+{
+ struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context;
+ struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm;
+
+ mutex_lock(&mxfrm->lock);
+ mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id);
+ mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id);
+ kfree(sa_ctx);
+ mxfrm->sa_ctx = NULL;
+ mutex_unlock(&mxfrm->lock);
+}
+
+static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev)
+{
+ return 0;
+}
+
+static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev,
+ struct mlx5_ipsec_obj_attrs *attrs,
+ u32 ipsec_id)
+{
+ u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
+ u64 modify_field_select = 0;
+ u64 general_obj_types;
+ void *obj;
+ int err;
+
+ if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED))
+ return 0;
+
+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+ return -EINVAL;
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (err) {
+ mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
+ ipsec_id, err);
+ return err;
+ }
+
+ obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
+ modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
+
+ /* esn */
+ if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
+ !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
+ return -EOPNOTSUPP;
+
+ obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
+ if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
+ MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
+
+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
+ struct mlx5_core_dev *mdev = xfrm->mdev;
+ struct mlx5_ipsec_esp_xfrm *mxfrm;
+
+ int err = 0;
+
+ if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
+ return 0;
+
+ if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs))
+ return -EOPNOTSUPP;
+
+ mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
+
+ mutex_lock(&mxfrm->lock);
+
+ if (!mxfrm->sa_ctx)
+ /* Not bound xfrm, change only sw attrs */
+ goto change_sw_xfrm_attrs;
+
+ /* need to add find and replace in ipsec_rhash_sa the sa_ctx */
+ /* modify device with new hw_sa */
+ ipsec_attrs.accel_flags = attrs->flags;
+ ipsec_attrs.esn_msb = attrs->esn;
+ err = mlx5_modify_ipsec_obj(mdev,
+ &ipsec_attrs,
+ mxfrm->sa_ctx->ipsec_obj_id);
+
+change_sw_xfrm_attrs:
+ if (!err)
+ memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));
+
+ mutex_unlock(&mxfrm->lock);
+ return err;
+}
+
+static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = {
+ .create_hw_context = mlx5_ipsec_offload_create_sa_ctx,
+ .free_hw_context = mlx5_ipsec_offload_delete_sa_ctx,
+ .init = mlx5_ipsec_offload_init,
+ .esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm,
+ .esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm,
+ .esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm,
+};
+
+static const struct mlx5_accel_ipsec_ops *
+mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev)
+{
+ if (!mlx5_ipsec_device_caps(mdev))
+ return NULL;
+
+ return &ipsec_offload_ops;
+}
+
+void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops;
+ int err = 0;
+
+ ipsec_ops = mlx5_ipsec_offload_ops(mdev);
+ if (!ipsec_ops || !ipsec_ops->init) {
+ mlx5_core_dbg(mdev, "IPsec ops is not supported\n");
+ return;
+ }
+
+ err = ipsec_ops->init(mdev);
+ if (err) {
+ mlx5_core_warn_once(
+ mdev, "Failed to start IPsec device, err = %d\n", err);
+ return;
+ }
+
+ mdev->ipsec_ops = ipsec_ops;
+}
+
+void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->cleanup)
+ return;
+
+ ipsec_ops->cleanup(mdev);
+}
+
+unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->counters_count)
+ return -EOPNOTSUPP;
+
+ return ipsec_ops->counters_count(mdev);
+}
+
+int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
+ unsigned int count)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->counters_read)
+ return -EOPNOTSUPP;
+
+ return ipsec_ops->counters_read(mdev, counters, count);
+}
+
+void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,
+ struct mlx5_accel_esp_xfrm *xfrm,
+ u32 *sa_handle)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+ __be32 saddr[4] = {}, daddr[4] = {};
+
+ if (!ipsec_ops || !ipsec_ops->create_hw_context)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (!xfrm->attrs.is_ipv6) {
+ saddr[3] = xfrm->attrs.saddr.a4;
+ daddr[3] = xfrm->attrs.daddr.a4;
+ } else {
+ memcpy(saddr, xfrm->attrs.saddr.a6, sizeof(saddr));
+ memcpy(daddr, xfrm->attrs.daddr.a6, sizeof(daddr));
+ }
+
+ return ipsec_ops->create_hw_context(mdev, xfrm, saddr, daddr,
+ xfrm->attrs.spi,
+ xfrm->attrs.is_ipv6, sa_handle);
+}
+
+void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->free_hw_context)
+ return;
+
+ ipsec_ops->free_hw_context(context);
+}
+
+struct mlx5_accel_esp_xfrm *
+mlx5_accel_esp_create_xfrm(struct mlx5_core_dev *mdev,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops;
+ struct mlx5_accel_esp_xfrm *xfrm;
+
+ if (!ipsec_ops || !ipsec_ops->esp_create_xfrm)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ xfrm = ipsec_ops->esp_create_xfrm(mdev, attrs, 0);
+ if (IS_ERR(xfrm))
+ return xfrm;
+
+ xfrm->mdev = mdev;
+ return xfrm;
+}
+EXPORT_SYMBOL_GPL(mlx5_accel_esp_create_xfrm);
+
+void mlx5_accel_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->esp_destroy_xfrm)
+ return;
+
+ ipsec_ops->esp_destroy_xfrm(xfrm);
+}
+EXPORT_SYMBOL_GPL(mlx5_accel_esp_destroy_xfrm);
+
+int mlx5_accel_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops;
+
+ if (!ipsec_ops || !ipsec_ops->esp_modify_xfrm)
+ return -EOPNOTSUPP;
+
+ return ipsec_ops->esp_modify_xfrm(xfrm, attrs);
+}
+EXPORT_SYMBOL_GPL(mlx5_accel_esp_modify_xfrm);
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
+
+#ifndef __MLX5_IPSEC_OFFLOAD_H__
+#define __MLX5_IPSEC_OFFLOAD_H__
+
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/accel.h>
+
+#ifdef CONFIG_MLX5_EN_IPSEC
+
+unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);
+int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
+ unsigned int count);
+
+void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,
+ struct mlx5_accel_esp_xfrm *xfrm,
+ u32 *sa_handle);
+void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context);
+
+void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);
+void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
+
+struct mlx5_accel_ipsec_ops {
+ unsigned int (*counters_count)(struct mlx5_core_dev *mdev);
+ int (*counters_read)(struct mlx5_core_dev *mdev, u64 *counters,
+ unsigned int count);
+ void *(*create_hw_context)(struct mlx5_core_dev *mdev,
+ struct mlx5_accel_esp_xfrm *xfrm,
+ const __be32 saddr[4], const __be32 daddr[4],
+ const __be32 spi, bool is_ipv6,
+ u32 *sa_handle);
+ void (*free_hw_context)(void *context);
+ int (*init)(struct mlx5_core_dev *mdev);
+ void (*cleanup)(struct mlx5_core_dev *mdev);
+ struct mlx5_accel_esp_xfrm *(*esp_create_xfrm)(
+ struct mlx5_core_dev *mdev,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs, u32 flags);
+ int (*esp_modify_xfrm)(struct mlx5_accel_esp_xfrm *xfrm,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs);
+ void (*esp_destroy_xfrm)(struct mlx5_accel_esp_xfrm *xfrm);
+};
+
+#else
+static inline void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) {}
+
+static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) {}
+#endif /* CONFIG_MLX5_EN_IPSEC */
+#endif /* __MLX5_IPSEC_OFFLOAD_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
@@ -34,7 +34,7 @@
#include <crypto/aead.h>
#include <net/xfrm.h>
#include <net/esp.h>
-#include "accel/ipsec_offload.h"
+#include "ipsec_offload.h"
#include "en_accel/ipsec_rxtx.h"
#include "en_accel/ipsec.h"
#include "en.h"
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c
@@ -35,7 +35,7 @@
#include <net/sock.h>
#include "en.h"
-#include "accel/ipsec_offload.h"
+#include "ipsec_offload.h"
#include "fpga/sdk.h"
#include "en_accel/ipsec.h"
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -48,7 +48,7 @@
#include "en_accel/ipsec.h"
#include "en_accel/en_accel.h"
#include "en_accel/ktls.h"
-#include "accel/ipsec_offload.h"
+#include "en_accel/ipsec_offload.h"
#include "lib/vxlan.h"
#include "lib/clock.h"
#include "en/port.h"
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -48,7 +48,7 @@
#include "en_rep.h"
#include "en/rep/tc.h"
#include "ipoib/ipoib.h"
-#include "accel/ipsec_offload.h"
+#include "en_accel/ipsec_offload.h"
#include "en_accel/ipsec_rxtx.h"
#include "en_accel/ktls_txrx.h"
#include "en/xdp.h"
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -62,7 +62,7 @@
#include "lib/mlx5.h"
#include "lib/tout.h"
#include "fpga/core.h"
-#include "accel/ipsec_offload.h"
+#include "en_accel/ipsec_offload.h"
#include "lib/clock.h"
#include "lib/vxlan.h"
#include "lib/geneve.h"