From: Maxim Mikityanskiy <maximmi@nvidia.com>
Date: Thu, 29 Sep 2022 00:21:50 -0700
Subject: net/mlx5e: Rename mlx5e_dma_info to prepare for removal of DMA
address
Patch-mainline: v6.1-rc1
Git-commit: 79008676d533b7ea92a5938b6526411ca5f2657f
References: jsc#PED-1549
The next commit will remove the DMA address from the struct currently
called mlx5e_dma_info, because the same value can be retrieved with
page_pool_get_dma_addr(page) in almost all cases, with the notable
exception of SHAMPO (HW GRO implementation) that modifies this address
on the fly, after the initial allocation.
To keep the SHAMPO logic intact, struct mlx5e_dma_info remains in the
SHAMPO code, consisting of addr and page (XSK is not compatible with
SHAMPO). The struct used in all other places is renamed to
mlx5e_alloc_unit, allowing the next commit to remove the addr field
without affecting SHAMPO.
The new name means "allocation unit", and it's more appropriate after
the field with the DMA address gets removed.
Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
---
drivers/net/ethernet/mellanox/mlx5/core/en.h | 15 -
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c | 4
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h | 8
drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 17 -
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 190 ++++++++++----------
5 files changed, 123 insertions(+), 111 deletions(-)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -474,7 +474,7 @@ struct mlx5e_txqsq {
cqe_ts_to_ns ptp_cyc2time;
} ____cacheline_aligned_in_smp;
-struct mlx5e_dma_info {
+struct mlx5e_alloc_unit {
dma_addr_t addr;
union {
struct page *page;
@@ -608,7 +608,7 @@ struct mlx5e_icosq {
} ____cacheline_aligned_in_smp;
struct mlx5e_wqe_frag_info {
- struct mlx5e_dma_info *di;
+ struct mlx5e_alloc_unit *au;
u32 offset;
bool last_in_page;
};
@@ -616,7 +616,7 @@ struct mlx5e_wqe_frag_info {
struct mlx5e_mpw_info {
u16 consumed_strides;
DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
- struct mlx5e_dma_info dma_info[];
+ struct mlx5e_alloc_unit alloc_units[];
};
#define MLX5E_MAX_RX_FRAGS 4
@@ -665,6 +665,11 @@ struct mlx5e_rq_frags_info {
u8 wqe_bulk;
};
+struct mlx5e_dma_info {
+ dma_addr_t addr;
+ struct page *page;
+};
+
struct mlx5e_shampo_hd {
u32 mkey;
struct mlx5e_dma_info *info;
@@ -690,7 +695,7 @@ struct mlx5e_rq {
struct {
struct mlx5_wq_cyc wq;
struct mlx5e_wqe_frag_info *frags;
- struct mlx5e_dma_info *di;
+ struct mlx5e_alloc_unit *alloc_units;
struct mlx5e_rq_frags_info info;
mlx5e_fp_skb_from_cqe skb_from_cqe;
} wqe;
@@ -1142,8 +1147,6 @@ void mlx5e_destroy_q_counters(struct mlx
int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
struct mlx5e_rq *drop_rq);
void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
-int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node);
-void mlx5e_free_di_list(struct mlx5e_rq *rq);
int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
@@ -30,7 +30,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_m
u32 head_offset,
u32 page_idx)
{
- struct xdp_buff *xdp = wi->dma_info[page_idx].xsk;
+ struct xdp_buff *xdp = wi->alloc_units[page_idx].xsk;
struct bpf_prog *prog;
/* Check packet size. Note LRO doesn't use linear SKB */
@@ -83,7 +83,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_l
struct mlx5e_wqe_frag_info *wi,
u32 cqe_bcnt)
{
- struct xdp_buff *xdp = wi->di->xsk;
+ struct xdp_buff *xdp = wi->au->xsk;
struct bpf_prog *prog;
/* wi->offset is not used in this function, because xdp->data and the
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -19,10 +19,10 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_l
u32 cqe_bcnt);
static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
+ struct mlx5e_alloc_unit *au)
{
- dma_info->xsk = xsk_buff_alloc(rq->xsk_pool);
- if (!dma_info->xsk)
+ au->xsk = xsk_buff_alloc(rq->xsk_pool);
+ if (!au->xsk)
return -ENOMEM;
/* Store the DMA address without headroom. In striding RQ case, we just
@@ -30,7 +30,7 @@ static inline int mlx5e_xsk_page_alloc_p
* when creating a WQE. In non-striding RQ case, headroom is accounted
* in mlx5e_alloc_rx_wqe.
*/
- dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk);
+ au->addr = xsk_buff_xdp_get_frame_dma(au->xsk);
return 0;
}
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -271,7 +271,7 @@ static int mlx5e_rq_alloc_mpwqe_info(str
int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
size_t alloc_size;
- alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, dma_info,
+ alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units,
rq->mpwqe.pages_per_wqe));
rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
@@ -433,7 +433,7 @@ static void mlx5e_init_frags_partition(s
struct mlx5e_wqe_frag_info *prev = NULL;
int i;
- next_frag.di = &rq->wqe.di[0];
+ next_frag.au = &rq->wqe.alloc_units[0];
for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
@@ -443,7 +443,7 @@ static void mlx5e_init_frags_partition(s
for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
- next_frag.di++;
+ next_frag.au++;
next_frag.offset = 0;
if (prev)
prev->last_in_page = true;
@@ -460,12 +460,13 @@ static void mlx5e_init_frags_partition(s
prev->last_in_page = true;
}
-int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
+static int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
{
int len = wq_sz << rq->wqe.info.log_num_frags;
- rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node);
- if (!rq->wqe.di)
+ rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)),
+ GFP_KERNEL, node);
+ if (!rq->wqe.alloc_units)
return -ENOMEM;
mlx5e_init_frags_partition(rq);
@@ -473,9 +474,9 @@ int mlx5e_init_di_list(struct mlx5e_rq *
return 0;
}
-void mlx5e_free_di_list(struct mlx5e_rq *rq)
+static void mlx5e_free_di_list(struct mlx5e_rq *rq)
{
- kvfree(rq->wqe.di);
+ kvfree(rq->wqe.alloc_units);
}
static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -77,7 +77,7 @@ const struct mlx5e_rx_handlers mlx5e_rx_
static struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i)
{
- size_t isz = struct_size(rq->mpwqe.info, dma_info, rq->mpwqe.pages_per_wqe);
+ size_t isz = struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe);
return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz));
}
@@ -250,8 +250,7 @@ static inline bool mlx5e_rx_cache_put(st
return true;
}
-static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
+static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au)
{
struct mlx5e_page_cache *cache = &rq->page_cache;
struct mlx5e_rq_stats *stats = rq->stats;
@@ -266,48 +265,44 @@ static inline bool mlx5e_rx_cache_get(st
return false;
}
- dma_info->page = cache->page_cache[cache->head];
- dma_info->addr = page_pool_get_dma_addr(dma_info->page);
+ au->page = cache->page_cache[cache->head];
+ au->addr = page_pool_get_dma_addr(au->page);
cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
stats->cache_reuse++;
- dma_sync_single_for_device(rq->pdev, dma_info->addr,
- /* Non-XSK always uses PAGE_SIZE. */
- PAGE_SIZE,
- DMA_FROM_DEVICE);
+ /* Non-XSK always uses PAGE_SIZE. */
+ dma_sync_single_for_device(rq->pdev, au->addr, PAGE_SIZE, DMA_FROM_DEVICE);
return true;
}
-static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
+static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au)
{
- if (mlx5e_rx_cache_get(rq, dma_info))
+ if (mlx5e_rx_cache_get(rq, au))
return 0;
- dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
- if (unlikely(!dma_info->page))
+ au->page = page_pool_dev_alloc_pages(rq->page_pool);
+ if (unlikely(!au->page))
return -ENOMEM;
/* Non-XSK always uses PAGE_SIZE. */
- dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE,
- rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
- page_pool_recycle_direct(rq->page_pool, dma_info->page);
- dma_info->page = NULL;
+ au->addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE,
+ rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ if (unlikely(dma_mapping_error(rq->pdev, au->addr))) {
+ page_pool_recycle_direct(rq->page_pool, au->page);
+ au->page = NULL;
return -ENOMEM;
}
- page_pool_set_dma_addr(dma_info->page, dma_info->addr);
+ page_pool_set_dma_addr(au->page, au->addr);
return 0;
}
-static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
+static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, struct mlx5e_alloc_unit *au)
{
if (rq->xsk_pool)
- return mlx5e_xsk_page_alloc_pool(rq, dma_info);
+ return mlx5e_xsk_page_alloc_pool(rq, au);
else
- return mlx5e_page_alloc_pool(rq, dma_info);
+ return mlx5e_page_alloc_pool(rq, au);
}
void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page)
@@ -335,7 +330,7 @@ void mlx5e_page_release_dynamic(struct m
}
static inline void mlx5e_page_release(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info,
+ struct mlx5e_alloc_unit *au,
bool recycle)
{
if (rq->xsk_pool)
@@ -343,9 +338,9 @@ static inline void mlx5e_page_release(st
* put into the Reuse Ring, because there is no way to return
* the page to the userspace when the interface goes down.
*/
- xsk_buff_free(dma_info->xsk);
+ xsk_buff_free(au->xsk);
else
- mlx5e_page_release_dynamic(rq, dma_info->page, recycle);
+ mlx5e_page_release_dynamic(rq, au->page, recycle);
}
static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
@@ -354,12 +349,12 @@ static inline int mlx5e_get_rx_frag(stru
int err = 0;
if (!frag->offset)
- /* On first frag (offset == 0), replenish page (dma_info actually).
- * Other frags that point to the same dma_info (with a different
+ /* On first frag (offset == 0), replenish page (alloc_unit actually).
+ * Other frags that point to the same alloc_unit (with a different
* offset) should just use the new one without replenishing again
* by themselves.
*/
- err = mlx5e_page_alloc(rq, frag->di);
+ err = mlx5e_page_alloc(rq, frag->au);
return err;
}
@@ -369,7 +364,7 @@ static inline void mlx5e_put_rx_frag(str
bool recycle)
{
if (frag->last_in_page)
- mlx5e_page_release(rq, frag->di, recycle);
+ mlx5e_page_release(rq, frag->au, recycle);
}
static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
@@ -392,7 +387,7 @@ static int mlx5e_alloc_rx_wqe(struct mlx
goto free_frags;
headroom = i == 0 ? rq->buff.headroom : 0;
- wqe->data[i].addr = cpu_to_be64(frag->di->addr +
+ wqe->data[i].addr = cpu_to_be64(frag->au->addr +
frag->offset + headroom);
}
@@ -458,36 +453,34 @@ free_wqes:
static inline void
mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
- struct mlx5e_dma_info *di, u32 frag_offset, u32 len,
+ struct mlx5e_alloc_unit *au, u32 frag_offset, u32 len,
unsigned int truesize)
{
- dma_sync_single_for_cpu(rq->pdev,
- di->addr + frag_offset,
+ dma_sync_single_for_cpu(rq->pdev, au->addr + frag_offset,
len, DMA_FROM_DEVICE);
- page_ref_inc(di->page);
+ page_ref_inc(au->page);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
- di->page, frag_offset, len, truesize);
+ au->page, frag_offset, len, truesize);
}
static inline void
mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
- struct mlx5e_dma_info *dma_info,
+ struct page *page, dma_addr_t addr,
int offset_from, int dma_offset, u32 headlen)
{
- const void *from = page_address(dma_info->page) + offset_from;
+ const void *from = page_address(page) + offset_from;
/* Aligning len to sizeof(long) optimizes memcpy performance */
unsigned int len = ALIGN(headlen, sizeof(long));
- dma_sync_single_for_cpu(pdev, dma_info->addr + dma_offset, len,
- DMA_FROM_DEVICE);
+ dma_sync_single_for_cpu(pdev, addr + dma_offset, len, DMA_FROM_DEVICE);
skb_copy_to_linear_data(skb, from, len);
}
static void
mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
{
+ struct mlx5e_alloc_unit *alloc_units = wi->alloc_units;
bool no_xdp_xmit;
- struct mlx5e_dma_info *dma_info = wi->dma_info;
int i;
/* A common case for AF_XDP. */
@@ -498,7 +491,7 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq,
for (i = 0; i < rq->mpwqe.pages_per_wqe; i++)
if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
- mlx5e_page_release(rq, &dma_info[i], recycle);
+ mlx5e_page_release(rq, &alloc_units[i], recycle);
}
static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
@@ -583,11 +576,13 @@ static int mlx5e_build_shampo_hd_umr(str
header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
if (!(header_offset & (PAGE_SIZE - 1))) {
- err = mlx5e_page_alloc(rq, dma_info);
+ struct mlx5e_alloc_unit au;
+
+ err = mlx5e_page_alloc(rq, &au);
if (unlikely(err))
goto err_unmap;
- addr = dma_info->addr;
- page = dma_info->page;
+ page = dma_info->page = au.page;
+ addr = dma_info->addr = page_pool_get_dma_addr(au.page);
} else {
dma_info->addr = addr + header_offset;
dma_info->page = page;
@@ -619,8 +614,12 @@ err_unmap:
while (--i >= 0) {
dma_info = &shampo->info[--index];
if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) {
+ struct mlx5e_alloc_unit au = {
+ .page = dma_info->page,
+ };
+
dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE);
- mlx5e_page_release(rq, dma_info, true);
+ mlx5e_page_release(rq, &au, true);
}
}
rq->stats->buff_alloc_err++;
@@ -669,7 +668,7 @@ static int mlx5e_alloc_rx_hd_mpwqe(struc
static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{
struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
- struct mlx5e_dma_info *dma_info = &wi->dma_info[0];
+ struct mlx5e_alloc_unit *au = &wi->alloc_units[0];
struct mlx5e_icosq *sq = rq->icosq;
struct mlx5_wq_cyc *wq = &sq->wq;
struct mlx5e_umr_wqe *umr_wqe;
@@ -698,22 +697,22 @@ static int mlx5e_alloc_rx_mpwqe(struct m
memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
if (unlikely(rq->mpwqe.unaligned)) {
- for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) {
- err = mlx5e_page_alloc(rq, dma_info);
+ for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) {
+ err = mlx5e_page_alloc(rq, au);
if (unlikely(err))
goto err_unmap;
umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
.key = rq->mkey_be,
- .va = cpu_to_be64(dma_info->addr),
+ .va = cpu_to_be64(au->addr),
};
}
} else {
- for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, dma_info++) {
- err = mlx5e_page_alloc(rq, dma_info);
+ for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) {
+ err = mlx5e_page_alloc(rq, au);
if (unlikely(err))
goto err_unmap;
umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
- .ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR),
+ .ptag = cpu_to_be64(au->addr | MLX5_EN_WR),
};
}
}
@@ -744,8 +743,8 @@ static int mlx5e_alloc_rx_mpwqe(struct m
err_unmap:
while (--i >= 0) {
- dma_info--;
- mlx5e_page_release(rq, dma_info, true);
+ au--;
+ mlx5e_page_release(rq, au, true);
}
err:
@@ -778,8 +777,12 @@ void mlx5e_shampo_dealloc_hd(struct mlx5
hd_info = &shampo->info[index];
hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE);
if (hd_info->page != deleted_page) {
+ struct mlx5e_alloc_unit au = {
+ .page = hd_info->page,
+ };
+
deleted_page = hd_info->page;
- mlx5e_page_release(rq, hd_info, false);
+ mlx5e_page_release(rq, &au, false);
}
}
@@ -1554,7 +1557,7 @@ static struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
u32 cqe_bcnt)
{
- struct mlx5e_dma_info *di = wi->di;
+ struct mlx5e_alloc_unit *au = wi->au;
u16 rx_headroom = rq->buff.headroom;
struct bpf_prog *prog;
struct sk_buff *skb;
@@ -1562,11 +1565,11 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_r
void *va, *data;
u32 frag_size;
- va = page_address(di->page) + wi->offset;
+ va = page_address(au->page) + wi->offset;
data = va + rx_headroom;
frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
- dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
+ dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset,
frag_size, DMA_FROM_DEVICE);
net_prefetch(data);
@@ -1576,7 +1579,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_r
net_prefetchw(va); /* xdp_frame data area */
mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
- if (mlx5e_xdp_handle(rq, di->page, prog, &xdp))
+ if (mlx5e_xdp_handle(rq, au->page, prog, &xdp))
return NULL; /* page/packet was consumed by XDP */
rx_headroom = xdp.data - xdp.data_hard_start;
@@ -1589,7 +1592,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_r
return NULL;
/* queue up for recycling/reuse */
- page_ref_inc(di->page);
+ page_ref_inc(au->page);
return skb;
}
@@ -1600,8 +1603,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
{
struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
struct mlx5e_wqe_frag_info *head_wi = wi;
+ struct mlx5e_alloc_unit *au = wi->au;
u16 rx_headroom = rq->buff.headroom;
- struct mlx5e_dma_info *di = wi->di;
struct skb_shared_info *sinfo;
u32 frag_consumed_bytes;
struct bpf_prog *prog;
@@ -1610,10 +1613,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
u32 truesize;
void *va;
- va = page_address(di->page) + wi->offset;
+ va = page_address(au->page) + wi->offset;
frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
- dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
+ dma_sync_single_range_for_cpu(rq->pdev, au->addr, wi->offset,
rq->buff.frame0_sz, DMA_FROM_DEVICE);
net_prefetchw(va); /* xdp_frame data area */
net_prefetch(va + rx_headroom);
@@ -1629,11 +1632,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
while (cqe_bcnt) {
skb_frag_t *frag;
- di = wi->di;
+ au = wi->au;
frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
- dma_sync_single_for_cpu(rq->pdev, di->addr + wi->offset,
+ dma_sync_single_for_cpu(rq->pdev, au->addr + wi->offset,
frag_consumed_bytes, DMA_FROM_DEVICE);
if (!xdp_buff_has_frags(&xdp)) {
@@ -1646,11 +1649,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
}
frag = &sinfo->frags[sinfo->nr_frags++];
- __skb_frag_set_page(frag, di->page);
+ __skb_frag_set_page(frag, au->page);
skb_frag_off_set(frag, wi->offset);
skb_frag_size_set(frag, frag_consumed_bytes);
- if (page_is_pfmemalloc(di->page))
+ if (page_is_pfmemalloc(au->page))
xdp_buff_set_frag_pfmemalloc(&xdp);
sinfo->xdp_frags_size += frag_consumed_bytes;
@@ -1661,10 +1664,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
wi++;
}
- di = head_wi->di;
+ au = head_wi->au;
prog = rcu_dereference(rq->xdp_prog);
- if (prog && mlx5e_xdp_handle(rq, di->page, prog, &xdp)) {
+ if (prog && mlx5e_xdp_handle(rq, au->page, prog, &xdp)) {
if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
int i;
@@ -1681,7 +1684,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5
if (unlikely(!skb))
return NULL;
- page_ref_inc(di->page);
+ page_ref_inc(au->page);
if (unlikely(xdp_buff_has_frags(&xdp))) {
int i;
@@ -1876,8 +1879,8 @@ const struct mlx5e_rx_handlers mlx5e_rx_
#endif
static void
-mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
- u32 data_bcnt, u32 data_offset)
+mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
+ struct mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset)
{
net_prefetchw(skb->data);
@@ -1891,12 +1894,12 @@ mlx5e_fill_skb_data(struct sk_buff *skb,
else
truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
- mlx5e_add_skb_frag(rq, skb, di, data_offset,
+ mlx5e_add_skb_frag(rq, skb, au, data_offset,
pg_consumed_bytes, truesize);
data_bcnt -= pg_consumed_bytes;
data_offset = 0;
- di++;
+ au++;
}
}
@@ -1904,11 +1907,11 @@ static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx)
{
+ struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
- struct mlx5e_dma_info *di = &wi->dma_info[page_idx];
u32 frag_offset = head_offset + headlen;
u32 byte_cnt = cqe_bcnt - headlen;
- struct mlx5e_dma_info *head_di = di;
+ struct mlx5e_alloc_unit *head_au = au;
struct sk_buff *skb;
skb = napi_alloc_skb(rq->cq.napi,
@@ -1922,13 +1925,14 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struc
/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
if (unlikely(frag_offset >= PAGE_SIZE)) {
- di++;
+ au++;
frag_offset -= PAGE_SIZE;
}
- mlx5e_fill_skb_data(skb, rq, di, byte_cnt, frag_offset);
+ mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset);
/* copy header */
- mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, head_offset, headlen);
+ mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, head_au->addr,
+ head_offset, head_offset, headlen);
/* skb linear part was allocated with headlen and aligned to long */
skb->tail += headlen;
skb->len += headlen;
@@ -1940,7 +1944,7 @@ static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx)
{
- struct mlx5e_dma_info *di = &wi->dma_info[page_idx];
+ struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
u16 rx_headroom = rq->buff.headroom;
struct bpf_prog *prog;
struct sk_buff *skb;
@@ -1954,11 +1958,11 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
return NULL;
}
- va = page_address(di->page) + head_offset;
+ va = page_address(au->page) + head_offset;
data = va + rx_headroom;
frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
- dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
+ dma_sync_single_range_for_cpu(rq->pdev, au->addr, head_offset,
frag_size, DMA_FROM_DEVICE);
net_prefetch(data);
@@ -1968,7 +1972,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
net_prefetchw(va); /* xdp_frame data area */
mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
- if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) {
+ if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) {
if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
return NULL; /* page/packet was consumed by XDP */
@@ -1984,7 +1988,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
return NULL;
/* queue up for recycling/reuse */
- page_ref_inc(di->page);
+ page_ref_inc(au->page);
return skb;
}
@@ -2029,7 +2033,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_r
}
prefetchw(skb->data);
- mlx5e_copy_skb_header(rq->pdev, skb, head,
+ mlx5e_copy_skb_header(rq->pdev, skb, head->page, head->addr,
head_offset + rx_headroom,
rx_headroom, head_size);
/* skb linear part was allocated with headlen and aligned to long */
@@ -2080,8 +2084,12 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx
u64 addr = shampo->info[header_index].addr;
if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
+ struct mlx5e_alloc_unit au = {
+ .page = shampo->info[header_index].page,
+ };
+
shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE);
- mlx5e_page_release(rq, &shampo->info[header_index], true);
+ mlx5e_page_release(rq, &au, true);
}
bitmap_clear(shampo->bitmap, header_index, 1);
}
@@ -2102,7 +2110,7 @@ static void mlx5e_handle_rx_cqe_mpwrq_sh
bool match = cqe->shampo.match;
struct mlx5e_rq_stats *stats = rq->stats;
struct mlx5e_rx_wqe_ll *wqe;
- struct mlx5e_dma_info *di;
+ struct mlx5e_alloc_unit *au;
struct mlx5e_mpw_info *wi;
struct mlx5_wq_ll *wq;
@@ -2152,8 +2160,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_sh
}
if (likely(head_size)) {
- di = &wi->dma_info[page_idx];
- mlx5e_fill_skb_data(*skb, rq, di, data_bcnt, data_offset);
+ au = &wi->alloc_units[page_idx];
+ mlx5e_fill_skb_data(*skb, rq, au, data_bcnt, data_offset);
}
mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb);