From: Tariq Toukan <tariqt@mellanox.com>
Date: Tue, 26 May 2020 13:58:09 +0300
Subject: net/mlx5e: kTLS, Improve TLS feature modularity
Patch-mainline: v5.9-rc1
Git-commit: 7d0d0d86ec6c67d3eb4f49d3bbccc2d8c02799cc
References: jsc#SLE-15172
Better separate the code into c/h files, so that kTLS internals
are exposed to the corresponding non-accel flow as follows:
- Necessary datapath functions are exposed via ktls_txrx.h.
- Necessary caps and configuration functions are exposed via ktls.h,
which became very small.
In addition, kTLS internal code sharing is done via ktls_utils.h,
which is not exposed to any non-accel file.
Add explicit WQE structures for the TLS static and progress
params, breaking the union of the static with UMR, and the progress
with PSV.
Generalize the API as a preparation for TLS RX offload support.
Move kTLS TX-specific code to the proper file.
Remove the inline tag for function in C files, let the compiler decide.
Use kzalloc/kfree for the priv_tx context.
Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
Reviewed-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/net/ethernet/mellanox/mlx5/core/Makefile | 3
drivers/net/ethernet/mellanox/mlx5/core/en.h | 14
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c | 64 --
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h | 107 ----
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c | 232 +++++-----
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c | 110 ++++
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h | 36 +
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h | 69 ++
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c | 14
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h | 7
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c | 14
drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h | 13
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 1
13 files changed, 386 insertions(+), 298 deletions(-)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -74,7 +74,8 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_
en_accel/ipsec_stats.o
mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \
- en_accel/ktls.o en_accel/ktls_tx.o en_accel/fs_tcp.o
+ en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \
+ en_accel/ktls_tx.o
mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \
steering/dr_matcher.o steering/dr_rule.o \
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -191,13 +191,8 @@ static inline int mlx5e_get_max_num_chan
struct mlx5e_tx_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
- union {
- struct {
- struct mlx5_wqe_eth_seg eth;
- struct mlx5_wqe_data_seg data[0];
- };
- u8 tls_progress_params_ctx[0];
- };
+ struct mlx5_wqe_eth_seg eth;
+ struct mlx5_wqe_data_seg data[0];
};
struct mlx5e_rx_wqe_ll {
@@ -213,10 +208,7 @@ struct mlx5e_umr_wqe {
struct mlx5_wqe_ctrl_seg ctrl;
struct mlx5_wqe_umr_ctrl_seg uctrl;
struct mlx5_mkey_seg mkc;
- union {
- struct mlx5_mtt inline_mtts[0];
- u8 tls_static_params_ctx[0];
- };
+ struct mlx5_mtt inline_mtts[0];
};
extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -3,31 +3,7 @@
#include "en.h"
#include "en_accel/ktls.h"
-
-u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq)
-{
- u16 num_dumps, stop_room = 0;
-
- num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
-
- stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_STATIC_WQEBBS);
- stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_PROGRESS_WQEBBS);
- stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS);
-
- return stop_room;
-}
-
-static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
-{
- u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
- void *tisc;
-
- tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
-
- MLX5_SET(tisc, tisc, tls_en, 1);
-
- return mlx5e_create_tis(mdev, in, tisn);
-}
+#include "en_accel/ktls_utils.h"
static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk,
enum tls_offload_ctx_dir direction,
@@ -35,8 +11,6 @@ static int mlx5e_ktls_add(struct net_dev
u32 start_offload_tcp_sn)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
- struct mlx5e_ktls_offload_context_tx *tx_priv;
- struct tls_context *tls_ctx = tls_get_ctx(sk);
struct mlx5_core_dev *mdev = priv->mdev;
int err;
@@ -46,31 +20,8 @@ static int mlx5e_ktls_add(struct net_dev
if (WARN_ON(!mlx5e_ktls_type_check(mdev, crypto_info)))
return -EOPNOTSUPP;
- tx_priv = kvzalloc(sizeof(*tx_priv), GFP_KERNEL);
- if (!tx_priv)
- return -ENOMEM;
-
- tx_priv->expected_seq = start_offload_tcp_sn;
- tx_priv->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
- mlx5e_set_ktls_tx_priv_ctx(tls_ctx, tx_priv);
-
- /* tc and underlay_qpn values are not in use for tls tis */
- err = mlx5e_ktls_create_tis(mdev, &tx_priv->tisn);
- if (err)
- goto create_tis_fail;
-
- err = mlx5_ktls_create_key(mdev, crypto_info, &tx_priv->key_id);
- if (err)
- goto encryption_key_create_fail;
-
- mlx5e_ktls_tx_offload_set_pending(tx_priv);
-
- return 0;
-
-encryption_key_create_fail:
- mlx5e_destroy_tis(priv->mdev, tx_priv->tisn);
-create_tis_fail:
- kvfree(tx_priv);
+ err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn);
+
return err;
}
@@ -78,13 +29,10 @@ static void mlx5e_ktls_del(struct net_de
struct tls_context *tls_ctx,
enum tls_offload_ctx_dir direction)
{
- struct mlx5e_priv *priv = netdev_priv(netdev);
- struct mlx5e_ktls_offload_context_tx *tx_priv =
- mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
+ if (direction != TLS_OFFLOAD_CTX_DIR_TX)
+ return;
- mlx5e_destroy_tis(priv->mdev, tx_priv->tisn);
- mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id);
- kvfree(tx_priv);
+ mlx5e_ktls_del_tx(netdev, tls_ctx);
}
static const struct tlsdev_ops mlx5e_ktls_ops = {
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -7,122 +7,15 @@
#include "en.h"
#ifdef CONFIG_MLX5_EN_TLS
-#include <net/tls.h>
-#include "accel/tls.h"
-#include "en_accel/tls_rxtx.h"
-
-#define MLX5E_KTLS_STATIC_UMR_WQE_SZ \
- (offsetof(struct mlx5e_umr_wqe, tls_static_params_ctx) + \
- MLX5_ST_SZ_BYTES(tls_static_params))
-#define MLX5E_KTLS_STATIC_WQEBBS \
- (DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_BB))
-
-#define MLX5E_KTLS_PROGRESS_WQE_SZ \
- (offsetof(struct mlx5e_tx_wqe, tls_progress_params_ctx) + \
- sizeof(struct mlx5_wqe_tls_progress_params_seg))
-#define MLX5E_KTLS_PROGRESS_WQEBBS \
- (DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB))
-
-struct mlx5e_dump_wqe {
- struct mlx5_wqe_ctrl_seg ctrl;
- struct mlx5_wqe_data_seg data;
-};
-
-#define MLX5E_TLS_FETCH_UMR_WQE(sq, pi) \
- ((struct mlx5e_umr_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_STATIC_UMR_WQE_SZ))
-#define MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi) \
- ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_PROGRESS_WQE_SZ))
-#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \
- ((struct mlx5e_dump_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, \
- sizeof(struct mlx5e_dump_wqe)))
-
-#define MLX5E_KTLS_DUMP_WQEBBS \
- (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
-
-enum {
- MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
- MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
- MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
-};
-
-enum {
- MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
- MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
- MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
-};
-
-struct mlx5e_ktls_offload_context_tx {
- struct tls_offload_context_tx *tx_ctx;
- struct tls12_crypto_info_aes_gcm_128 crypto_info;
- u32 expected_seq;
- u32 tisn;
- u32 key_id;
- bool ctx_post_pending;
-};
-
-struct mlx5e_ktls_offload_context_tx_shadow {
- struct tls_offload_context_tx tx_ctx;
- struct mlx5e_ktls_offload_context_tx *priv_tx;
-};
-
-static inline void
-mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
- struct mlx5e_ktls_offload_context_tx *priv_tx)
-{
- struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
- struct mlx5e_ktls_offload_context_tx_shadow *shadow;
-
- BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
-
- shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
-
- shadow->priv_tx = priv_tx;
- priv_tx->tx_ctx = tx_ctx;
-}
-
-static inline struct mlx5e_ktls_offload_context_tx *
-mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
-{
- struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
- struct mlx5e_ktls_offload_context_tx_shadow *shadow;
-
- BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
-
- shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
-
- return shadow->priv_tx;
-}
void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv);
-void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx);
-bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq,
- struct sk_buff *skb, int datalen,
- struct mlx5e_accel_tx_tls_state *state);
-void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
- struct mlx5e_tx_wqe_info *wi,
- u32 *dma_fifo_cc);
-u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq);
-
-static inline u8
-mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags,
- unsigned int sync_len)
-{
- /* Given the MTU and sync_len, calculates an upper bound for the
- * number of DUMP WQEs needed for the TX resync of a record.
- */
- return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu);
-}
#else
static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv)
{
}
-static inline void
-mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
- struct mlx5e_tx_wqe_info *wi,
- u32 *dma_fifo_cc) {}
#endif
#endif /* __MLX5E_TLS_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
@@ -1,109 +1,149 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2019 Mellanox Technologies.
-#include <linux/tls.h>
-#include "en.h"
-#include "en/txrx.h"
-#include "en_accel/ktls.h"
+#include "en_accel/ktls_txrx.h"
+#include "en_accel/ktls_utils.h"
-enum {
- MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+struct mlx5e_dump_wqe {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_data_seg data;
};
-enum {
- MLX5E_ENCRYPTION_STANDARD_TLS = 0x1,
+#define MLX5E_KTLS_DUMP_WQEBBS \
+ (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
+
+static u8
+mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags,
+ unsigned int sync_len)
+{
+ /* Given the MTU and sync_len, calculates an upper bound for the
+ * number of DUMP WQEs needed for the TX resync of a record.
+ */
+ return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu);
+}
+
+u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq)
+{
+ u16 num_dumps, stop_room = 0;
+
+ num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
+
+ stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS);
+ stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS);
+ stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS);
+
+ return stop_room;
+}
+
+static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
+{
+ u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
+ void *tisc;
+
+ tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
+
+ MLX5_SET(tisc, tisc, tls_en, 1);
+
+ return mlx5e_create_tis(mdev, in, tisn);
+}
+
+struct mlx5e_ktls_offload_context_tx {
+ struct tls_offload_context_tx *tx_ctx;
+ struct tls12_crypto_info_aes_gcm_128 crypto_info;
+ u32 expected_seq;
+ u32 tisn;
+ u32 key_id;
+ bool ctx_post_pending;
};
-#define EXTRACT_INFO_FIELDS do { \
- salt = info->salt; \
- rec_seq = info->rec_seq; \
- salt_sz = sizeof(info->salt); \
- rec_seq_sz = sizeof(info->rec_seq); \
-} while (0)
+struct mlx5e_ktls_offload_context_tx_shadow {
+ struct tls_offload_context_tx tx_ctx;
+ struct mlx5e_ktls_offload_context_tx *priv_tx;
+};
static void
-fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
+mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
+ struct mlx5e_ktls_offload_context_tx *priv_tx)
{
- struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
- char *initial_rn, *gcm_iv;
- u16 salt_sz, rec_seq_sz;
- char *salt, *rec_seq;
- u8 tls_version;
-
- EXTRACT_INFO_FIELDS;
-
- gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
- initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
-
- memcpy(gcm_iv, salt, salt_sz);
- memcpy(initial_rn, rec_seq, rec_seq_sz);
-
- tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
-
- MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
- MLX5_SET(tls_static_params, ctx, const_1, 1);
- MLX5_SET(tls_static_params, ctx, const_2, 2);
- MLX5_SET(tls_static_params, ctx, encryption_standard,
- MLX5E_ENCRYPTION_STANDARD_TLS);
- MLX5_SET(tls_static_params, ctx, dek_index, priv_tx->key_id);
+ struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
+ struct mlx5e_ktls_offload_context_tx_shadow *shadow;
+
+ BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
+
+ shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
+
+ shadow->priv_tx = priv_tx;
+ priv_tx->tx_ctx = tx_ctx;
}
-static void
-build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn,
- struct mlx5e_ktls_offload_context_tx *priv_tx,
- bool fence)
-{
- struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
- struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
-
-#define STATIC_PARAMS_DS_CNT \
- DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_DS)
-
- cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR |
- (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24));
- cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
- STATIC_PARAMS_DS_CNT);
- cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
- cseg->tis_tir_num = cpu_to_be32(priv_tx->tisn << 8);
+static struct mlx5e_ktls_offload_context_tx *
+mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
+{
+ struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx);
+ struct mlx5e_ktls_offload_context_tx_shadow *shadow;
+
+ BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX);
- ucseg->flags = MLX5_UMR_INLINE;
- ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
+ shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx;
- fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx);
+ return shadow->priv_tx;
}
-static void
-fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
+int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
+ struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn)
{
- struct mlx5_wqe_tls_progress_params_seg *params;
+ struct mlx5e_ktls_offload_context_tx *priv_tx;
+ struct tls_context *tls_ctx;
+ struct mlx5_core_dev *mdev;
+ struct mlx5e_priv *priv;
+ int err;
+
+ tls_ctx = tls_get_ctx(sk);
+ priv = netdev_priv(netdev);
+ mdev = priv->mdev;
- params = ctx;
+ priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL);
+ if (!priv_tx)
+ return -ENOMEM;
- params->tis_tir_num = cpu_to_be32(priv_tx->tisn);
- MLX5_SET(tls_progress_params, params->ctx, record_tracker_state,
- MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
- MLX5_SET(tls_progress_params, params->ctx, auth_state,
- MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
+ err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id);
+ if (err)
+ goto err_create_key;
+
+ priv_tx->expected_seq = start_offload_tcp_sn;
+ priv_tx->crypto_info =
+ *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+
+ mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);
+
+ err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn);
+ if (err)
+ goto err_create_tis;
+
+ priv_tx->ctx_post_pending = true;
+
+ return 0;
+
+err_create_tis:
+ mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
+err_create_key:
+ kfree(priv_tx);
+ return err;
}
-static void
-build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn,
- struct mlx5e_ktls_offload_context_tx *priv_tx,
- bool fence)
-{
- struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
-
-#define PROGRESS_PARAMS_DS_CNT \
- DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_DS)
-
- cseg->opmod_idx_opcode =
- cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV |
- (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24));
- cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
- PROGRESS_PARAMS_DS_CNT);
- cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
+void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
+{
+ struct mlx5e_ktls_offload_context_tx *priv_tx;
+ struct mlx5_core_dev *mdev;
+ struct mlx5e_priv *priv;
+
+ priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
+ priv = netdev_priv(netdev);
+ mdev = priv->mdev;
- fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx);
+ mlx5e_destroy_tis(mdev, priv_tx->tisn);
+ mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
+ kfree(priv_tx);
}
static void tx_fill_wi(struct mlx5e_txqsq *sq,
@@ -119,11 +159,6 @@ static void tx_fill_wi(struct mlx5e_txqs
};
}
-void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
-{
- priv_tx->ctx_post_pending = true;
-}
-
static bool
mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
{
@@ -139,12 +174,15 @@ post_static_params(struct mlx5e_txqsq *s
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
{
- u16 pi, num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS;
- struct mlx5e_umr_wqe *umr_wqe;
+ struct mlx5e_set_tls_static_params_wqe *wqe;
+ u16 pi, num_wqebbs;
+ num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
- umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi);
- build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence);
+ wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
+ mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
+ priv_tx->tisn, priv_tx->key_id, fence,
+ TLS_OFFLOAD_CTX_DIR_TX);
tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
sq->pc += num_wqebbs;
}
@@ -154,12 +192,14 @@ post_progress_params(struct mlx5e_txqsq
struct mlx5e_ktls_offload_context_tx *priv_tx,
bool fence)
{
- u16 pi, num_wqebbs = MLX5E_KTLS_PROGRESS_WQEBBS;
- struct mlx5e_tx_wqe *wqe;
+ struct mlx5e_set_tls_progress_params_wqe *wqe;
+ u16 pi, num_wqebbs;
+ num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
- wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi);
- build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence);
+ wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
+ mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence,
+ TLS_OFFLOAD_CTX_DIR_TX);
tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
sq->pc += num_wqebbs;
}
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
+
+#include "en_accel/ktls_txrx.h"
+#include "en_accel/ktls_utils.h"
+
+enum {
+ MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+};
+
+enum {
+ MLX5E_ENCRYPTION_STANDARD_TLS = 0x1,
+};
+
+#define EXTRACT_INFO_FIELDS do { \
+ salt = info->salt; \
+ rec_seq = info->rec_seq; \
+ salt_sz = sizeof(info->salt); \
+ rec_seq_sz = sizeof(info->rec_seq); \
+} while (0)
+
+static void
+fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
+ struct tls12_crypto_info_aes_gcm_128 *info,
+ u32 key_id)
+{
+ char *initial_rn, *gcm_iv;
+ u16 salt_sz, rec_seq_sz;
+ char *salt, *rec_seq;
+ u8 tls_version;
+ u8 *ctx;
+
+ ctx = params->ctx;
+
+ EXTRACT_INFO_FIELDS;
+
+ gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
+ initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
+
+ memcpy(gcm_iv, salt, salt_sz);
+ memcpy(initial_rn, rec_seq, rec_seq_sz);
+
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+
+ MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
+ MLX5_SET(tls_static_params, ctx, const_1, 1);
+ MLX5_SET(tls_static_params, ctx, const_2, 2);
+ MLX5_SET(tls_static_params, ctx, encryption_standard,
+ MLX5E_ENCRYPTION_STANDARD_TLS);
+ MLX5_SET(tls_static_params, ctx, dek_index, key_id);
+}
+
+void
+mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
+ u16 pc, u32 sqn,
+ struct tls12_crypto_info_aes_gcm_128 *info,
+ u32 tis_tir_num, u32 key_id,
+ bool fence, enum tls_offload_ctx_dir direction)
+{
+ struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
+ struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
+
+#define STATIC_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)
+
+ cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR |
+ (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24));
+ cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
+ STATIC_PARAMS_DS_CNT);
+ cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
+ cseg->tis_tir_num = cpu_to_be32(tis_tir_num << 8);
+
+ ucseg->flags = MLX5_UMR_INLINE;
+ ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
+
+ fill_static_params(&wqe->params, info, key_id);
+}
+
+static void
+fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num)
+{
+ u8 *ctx = params->ctx;
+
+ params->tis_tir_num = cpu_to_be32(tis_tir_num);
+
+ MLX5_SET(tls_progress_params, ctx, record_tracker_state,
+ MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
+ MLX5_SET(tls_progress_params, ctx, auth_state,
+ MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
+}
+
+void
+mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe,
+ u16 pc, u32 sqn,
+ u32 tis_tir_num, bool fence,
+ enum tls_offload_ctx_dir direction)
+{
+ struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
+
+#define PROGRESS_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)
+
+ cseg->opmod_idx_opcode =
+ cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV |
+ (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24));
+ cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
+ PROGRESS_PARAMS_DS_CNT);
+ cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
+
+ fill_progress_params(&wqe->params, tis_tir_num);
+}
+
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
+
+#ifndef __MLX5E_KTLS_TXRX_H__
+#define __MLX5E_KTLS_TXRX_H__
+
+#ifdef CONFIG_MLX5_EN_TLS
+
+#include <net/tls.h>
+#include "en.h"
+#include "en/txrx.h"
+
+struct mlx5e_accel_tx_tls_state {
+ u32 tls_tisn;
+};
+
+u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq);
+
+bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq,
+ struct sk_buff *skb, int datalen,
+ struct mlx5e_accel_tx_tls_state *state);
+
+void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
+ struct mlx5e_tx_wqe_info *wi,
+ u32 *dma_fifo_cc);
+#else
+static inline void
+mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
+ struct mlx5e_tx_wqe_info *wi,
+ u32 *dma_fifo_cc)
+{
+}
+
+#endif /* CONFIG_MLX5_EN_TLS */
+
+#endif /* __MLX5E_TLS_TXRX_H__ */
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
+
+#ifndef __MLX5E_KTLS_UTILS_H__
+#define __MLX5E_KTLS_UTILS_H__
+
+#include <net/tls.h>
+#include "en.h"
+#include "accel/tls.h"
+
+enum {
+ MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
+ MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
+ MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
+};
+
+enum {
+ MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
+ MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
+ MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
+};
+
+int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
+ struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn);
+void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx);
+
+struct mlx5e_set_tls_static_params_wqe {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_umr_ctrl_seg uctrl;
+ struct mlx5_mkey_seg mkc;
+ struct mlx5_wqe_tls_static_params_seg params;
+};
+
+struct mlx5e_set_tls_progress_params_wqe {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_tls_progress_params_seg params;
+};
+
+#define MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS \
+ (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_static_params_wqe), MLX5_SEND_WQE_BB))
+
+#define MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS \
+ (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_progress_params_wqe), MLX5_SEND_WQE_BB))
+
+#define MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi) \
+ ((struct mlx5e_set_tls_static_params_wqe *)\
+ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_static_params_wqe)))
+
+#define MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi) \
+ ((struct mlx5e_set_tls_progress_params_wqe *)\
+ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_progress_params_wqe)))
+
+#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \
+ ((struct mlx5e_dump_wqe *)\
+ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_dump_wqe)))
+
+void
+mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
+ u16 pc, u32 sqn,
+ struct tls12_crypto_info_aes_gcm_128 *info,
+ u32 tis_tir_num, u32 key_id,
+ bool fence, enum tls_offload_ctx_dir direction);
+void
+mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe,
+ u16 pc, u32 sqn,
+ u32 tis_tir_num, bool fence,
+ enum tls_offload_ctx_dir direction);
+
+#endif /* __MLX5E_TLS_UTILS_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
@@ -240,17 +240,3 @@ void mlx5e_tls_cleanup(struct mlx5e_priv
kfree(tls);
priv->tls = NULL;
}
-
-u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
-{
- struct mlx5_core_dev *mdev = sq->channel->mdev;
-
- if (!mlx5_accel_is_tls_device(mdev))
- return 0;
-
- if (MLX5_CAP_GEN(mdev, tls_tx))
- return mlx5e_ktls_get_stop_room(sq);
-
- /* Resync SKB. */
- return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
-}
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
@@ -94,8 +94,6 @@ int mlx5e_tls_get_count(struct mlx5e_pri
int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data);
int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data);
-u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq);
-
#else
static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv)
@@ -110,11 +108,6 @@ static inline int mlx5e_tls_get_count(st
static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; }
static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; }
-static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
-{
- return 0;
-}
-
#endif
#endif /* __MLX5E_TLS_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
@@ -387,3 +387,17 @@ void mlx5e_tls_handle_rx_skb(struct net_
remove_metadata_hdr(skb);
*cqe_bcnt -= MLX5E_METADATA_ETHER_LEN;
}
+
+u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
+{
+ struct mlx5_core_dev *mdev = sq->channel->mdev;
+
+ if (!mlx5_accel_is_tls_device(mdev))
+ return 0;
+
+ if (MLX5_CAP_GEN(mdev, tls_tx))
+ return mlx5e_ktls_get_stop_room(sq);
+
+ /* Resync SKB. */
+ return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
+}
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
@@ -34,15 +34,15 @@
#ifndef __MLX5E_TLS_RXTX_H__
#define __MLX5E_TLS_RXTX_H__
+#include "en_accel/ktls_txrx.h"
+
#ifdef CONFIG_MLX5_EN_TLS
#include <linux/skbuff.h>
#include "en.h"
#include "en/txrx.h"
-struct mlx5e_accel_tx_tls_state {
- u32 tls_tisn;
-};
+u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq);
bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq,
struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state);
@@ -52,6 +52,13 @@ void mlx5e_tls_handle_tx_wqe(struct mlx5
void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb,
u32 *cqe_bcnt);
+#else
+
+static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq)
+{
+ return 0;
+}
+
#endif /* CONFIG_MLX5_EN_TLS */
#endif /* __MLX5E_TLS_RXTX_H__ */
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -38,7 +38,6 @@
#include "en/txrx.h"
#include "ipoib/ipoib.h"
#include "en_accel/en_accel.h"
-#include "en_accel/ktls.h"
#include "lib/clock.h"
static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)