|
Jiri Slaby |
256201 |
From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>
|
|
Jiri Slaby |
256201 |
Date: Tue, 11 Apr 2023 11:10:54 -0400
|
|
Jiri Slaby |
256201 |
Subject: [PATCH] maple_tree: add RCU lock checking to rcu callback functions
|
|
Jiri Slaby |
256201 |
References: bsc#1012628
|
|
Jiri Slaby |
256201 |
Patch-mainline: 6.2.11
|
|
Jiri Slaby |
256201 |
Git-commit: 790e1fa86b340c2bd4a327e01c161f7a1ad885f6
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
commit 790e1fa86b340c2bd4a327e01c161f7a1ad885f6 upstream.
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
Dereferencing RCU objects within the RCU callback without the RCU check
|
|
Jiri Slaby |
256201 |
has caused lockdep to complain. Fix the RCU dereferencing by using the
|
|
Jiri Slaby |
256201 |
RCU callback lock to ensure the operation is safe.
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
Also stop creating a new lock to use for dereferencing during destruction
|
|
Jiri Slaby |
256201 |
of the tree or subtree. Instead, pass through a pointer to the tree that
|
|
Jiri Slaby |
256201 |
has the lock that is held for RCU dereferencing checking. It also does
|
|
Jiri Slaby |
256201 |
not make sense to use the maple state in the freeing scenario as the tree
|
|
Jiri Slaby |
256201 |
walk is a special case where the tree no longer has the normal encodings
|
|
Jiri Slaby |
256201 |
and parent pointers.
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
Link: https://lkml.kernel.org/r/20230227173632.3292573-8-surenb@google.com
|
|
Jiri Slaby |
256201 |
Fixes: 54a611b60590 ("Maple Tree: add new data structure")
|
|
Jiri Slaby |
256201 |
Cc: stable@vger.kernel.org
|
|
Jiri Slaby |
256201 |
Reported-by: Suren Baghdasaryan <surenb@google.com>
|
|
Jiri Slaby |
256201 |
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
|
|
Jiri Slaby |
256201 |
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
|
|
Jiri Slaby |
256201 |
Signed-off-by: Jiri Slaby <jslaby@suse.cz>
|
|
Jiri Slaby |
256201 |
---
|
|
Jiri Slaby |
256201 |
lib/maple_tree.c | 188 ++++++++++++++++++++++++-----------------------
|
|
Jiri Slaby |
256201 |
1 file changed, 96 insertions(+), 92 deletions(-)
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
|
|
Jiri Slaby |
256201 |
index 1ec872809..fb452873 100644
|
|
Jiri Slaby |
256201 |
--- a/lib/maple_tree.c
|
|
Jiri Slaby |
256201 |
+++ b/lib/maple_tree.c
|
|
Jiri Slaby |
256201 |
@@ -819,6 +819,11 @@ static inline void *mt_slot(const struct maple_tree *mt,
|
|
Jiri Slaby |
256201 |
return rcu_dereference_check(slots[offset], mt_locked(mt));
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
+static inline void *mt_slot_locked(struct maple_tree *mt, void __rcu **slots,
|
|
Jiri Slaby |
256201 |
+ unsigned char offset)
|
|
Jiri Slaby |
256201 |
+{
|
|
Jiri Slaby |
256201 |
+ return rcu_dereference_protected(slots[offset], mt_locked(mt));
|
|
Jiri Slaby |
256201 |
+}
|
|
Jiri Slaby |
256201 |
/*
|
|
Jiri Slaby |
256201 |
* mas_slot_locked() - Get the slot value when holding the maple tree lock.
|
|
Jiri Slaby |
256201 |
* @mas: The maple state
|
|
Jiri Slaby |
256201 |
@@ -830,7 +835,7 @@ static inline void *mt_slot(const struct maple_tree *mt,
|
|
Jiri Slaby |
256201 |
static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots,
|
|
Jiri Slaby |
256201 |
unsigned char offset)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
- return rcu_dereference_protected(slots[offset], mt_locked(mas->tree));
|
|
Jiri Slaby |
256201 |
+ return mt_slot_locked(mas->tree, slots, offset);
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
/*
|
|
Jiri Slaby |
256201 |
@@ -902,34 +907,35 @@ static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt,
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
/*
|
|
Jiri Slaby |
256201 |
- * mas_clear_meta() - clear the metadata information of a node, if it exists
|
|
Jiri Slaby |
256201 |
- * @mas: The maple state
|
|
Jiri Slaby |
256201 |
+ * mt_clear_meta() - clear the metadata information of a node, if it exists
|
|
Jiri Slaby |
256201 |
+ * @mt: The maple tree
|
|
Jiri Slaby |
256201 |
* @mn: The maple node
|
|
Jiri Slaby |
256201 |
- * @mt: The maple node type
|
|
Jiri Slaby |
256201 |
+ * @type: The maple node type
|
|
Jiri Slaby |
256201 |
* @offset: The offset of the highest sub-gap in this node.
|
|
Jiri Slaby |
256201 |
* @end: The end of the data in this node.
|
|
Jiri Slaby |
256201 |
*/
|
|
Jiri Slaby |
256201 |
-static inline void mas_clear_meta(struct ma_state *mas, struct maple_node *mn,
|
|
Jiri Slaby |
256201 |
- enum maple_type mt)
|
|
Jiri Slaby |
256201 |
+static inline void mt_clear_meta(struct maple_tree *mt, struct maple_node *mn,
|
|
Jiri Slaby |
256201 |
+ enum maple_type type)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
struct maple_metadata *meta;
|
|
Jiri Slaby |
256201 |
unsigned long *pivots;
|
|
Jiri Slaby |
256201 |
void __rcu **slots;
|
|
Jiri Slaby |
256201 |
void *next;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- switch (mt) {
|
|
Jiri Slaby |
256201 |
+ switch (type) {
|
|
Jiri Slaby |
256201 |
case maple_range_64:
|
|
Jiri Slaby |
256201 |
pivots = mn->mr64.pivot;
|
|
Jiri Slaby |
256201 |
if (unlikely(pivots[MAPLE_RANGE64_SLOTS - 2])) {
|
|
Jiri Slaby |
256201 |
slots = mn->mr64.slot;
|
|
Jiri Slaby |
256201 |
- next = mas_slot_locked(mas, slots,
|
|
Jiri Slaby |
256201 |
- MAPLE_RANGE64_SLOTS - 1);
|
|
Jiri Slaby |
256201 |
- if (unlikely((mte_to_node(next) && mte_node_type(next))))
|
|
Jiri Slaby |
256201 |
- return; /* The last slot is a node, no metadata */
|
|
Jiri Slaby |
256201 |
+ next = mt_slot_locked(mt, slots,
|
|
Jiri Slaby |
256201 |
+ MAPLE_RANGE64_SLOTS - 1);
|
|
Jiri Slaby |
256201 |
+ if (unlikely((mte_to_node(next) &&
|
|
Jiri Slaby |
256201 |
+ mte_node_type(next))))
|
|
Jiri Slaby |
256201 |
+ return; /* no metadata, could be node */
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
fallthrough;
|
|
Jiri Slaby |
256201 |
case maple_arange_64:
|
|
Jiri Slaby |
256201 |
- meta = ma_meta(mn, mt);
|
|
Jiri Slaby |
256201 |
+ meta = ma_meta(mn, type);
|
|
Jiri Slaby |
256201 |
break;
|
|
Jiri Slaby |
256201 |
default:
|
|
Jiri Slaby |
256201 |
return;
|
|
Jiri Slaby |
256201 |
@@ -5477,7 +5483,7 @@ static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min,
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
/*
|
|
Jiri Slaby |
256201 |
- * mas_dead_leaves() - Mark all leaves of a node as dead.
|
|
Jiri Slaby |
256201 |
+ * mte_dead_leaves() - Mark all leaves of a node as dead.
|
|
Jiri Slaby |
256201 |
* @mas: The maple state
|
|
Jiri Slaby |
256201 |
* @slots: Pointer to the slot array
|
|
Jiri Slaby |
256201 |
* @type: The maple node type
|
|
Jiri Slaby |
256201 |
@@ -5487,16 +5493,16 @@ static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min,
|
|
Jiri Slaby |
256201 |
* Return: The number of leaves marked as dead.
|
|
Jiri Slaby |
256201 |
*/
|
|
Jiri Slaby |
256201 |
static inline
|
|
Jiri Slaby |
256201 |
-unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots,
|
|
Jiri Slaby |
256201 |
- enum maple_type mt)
|
|
Jiri Slaby |
256201 |
+unsigned char mte_dead_leaves(struct maple_enode *enode, struct maple_tree *mt,
|
|
Jiri Slaby |
256201 |
+ void __rcu **slots)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
struct maple_node *node;
|
|
Jiri Slaby |
256201 |
enum maple_type type;
|
|
Jiri Slaby |
256201 |
void *entry;
|
|
Jiri Slaby |
256201 |
int offset;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- for (offset = 0; offset < mt_slots[mt]; offset++) {
|
|
Jiri Slaby |
256201 |
- entry = mas_slot_locked(mas, slots, offset);
|
|
Jiri Slaby |
256201 |
+ for (offset = 0; offset < mt_slot_count(enode); offset++) {
|
|
Jiri Slaby |
256201 |
+ entry = mt_slot(mt, slots, offset);
|
|
Jiri Slaby |
256201 |
type = mte_node_type(entry);
|
|
Jiri Slaby |
256201 |
node = mte_to_node(entry);
|
|
Jiri Slaby |
256201 |
/* Use both node and type to catch LE & BE metadata */
|
|
Jiri Slaby |
256201 |
@@ -5511,162 +5517,160 @@ unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots,
|
|
Jiri Slaby |
256201 |
return offset;
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
-static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset)
|
|
Jiri Slaby |
256201 |
+/**
|
|
Jiri Slaby |
256201 |
+ * mte_dead_walk() - Walk down a dead tree to just before the leaves
|
|
Jiri Slaby |
256201 |
+ * @enode: The maple encoded node
|
|
Jiri Slaby |
256201 |
+ * @offset: The starting offset
|
|
Jiri Slaby |
256201 |
+ *
|
|
Jiri Slaby |
256201 |
+ * Note: This can only be used from the RCU callback context.
|
|
Jiri Slaby |
256201 |
+ */
|
|
Jiri Slaby |
256201 |
+static void __rcu **mte_dead_walk(struct maple_enode **enode, unsigned char offset)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
- struct maple_node *next;
|
|
Jiri Slaby |
256201 |
+ struct maple_node *node, *next;
|
|
Jiri Slaby |
256201 |
void __rcu **slots = NULL;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- next = mas_mn(mas);
|
|
Jiri Slaby |
256201 |
+ next = mte_to_node(*enode);
|
|
Jiri Slaby |
256201 |
do {
|
|
Jiri Slaby |
256201 |
- mas->node = mt_mk_node(next, next->type);
|
|
Jiri Slaby |
256201 |
- slots = ma_slots(next, next->type);
|
|
Jiri Slaby |
256201 |
- next = mas_slot_locked(mas, slots, offset);
|
|
Jiri Slaby |
256201 |
+ *enode = ma_enode_ptr(next);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(*enode);
|
|
Jiri Slaby |
256201 |
+ slots = ma_slots(node, node->type);
|
|
Jiri Slaby |
256201 |
+ next = rcu_dereference_protected(slots[offset],
|
|
Jiri Slaby |
256201 |
+ lock_is_held(&rcu_callback_map));
|
|
Jiri Slaby |
256201 |
offset = 0;
|
|
Jiri Slaby |
256201 |
} while (!ma_is_leaf(next->type));
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
return slots;
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
+/**
|
|
Jiri Slaby |
256201 |
+ * mt_free_walk() - Walk & free a tree in the RCU callback context
|
|
Jiri Slaby |
256201 |
+ * @head: The RCU head that's within the node.
|
|
Jiri Slaby |
256201 |
+ *
|
|
Jiri Slaby |
256201 |
+ * Note: This can only be used from the RCU callback context.
|
|
Jiri Slaby |
256201 |
+ */
|
|
Jiri Slaby |
256201 |
static void mt_free_walk(struct rcu_head *head)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
void __rcu **slots;
|
|
Jiri Slaby |
256201 |
struct maple_node *node, *start;
|
|
Jiri Slaby |
256201 |
- struct maple_tree mt;
|
|
Jiri Slaby |
256201 |
+ struct maple_enode *enode;
|
|
Jiri Slaby |
256201 |
unsigned char offset;
|
|
Jiri Slaby |
256201 |
enum maple_type type;
|
|
Jiri Slaby |
256201 |
- MA_STATE(mas, &mt, 0, 0);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
node = container_of(head, struct maple_node, rcu);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
if (ma_is_leaf(node->type))
|
|
Jiri Slaby |
256201 |
goto free_leaf;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- mt_init_flags(&mt, node->ma_flags);
|
|
Jiri Slaby |
256201 |
- mas_lock(&mas;;
|
|
Jiri Slaby |
256201 |
start = node;
|
|
Jiri Slaby |
256201 |
- mas.node = mt_mk_node(node, node->type);
|
|
Jiri Slaby |
256201 |
- slots = mas_dead_walk(&mas, 0);
|
|
Jiri Slaby |
256201 |
- node = mas_mn(&mas;;
|
|
Jiri Slaby |
256201 |
+ enode = mt_mk_node(node, node->type);
|
|
Jiri Slaby |
256201 |
+ slots = mte_dead_walk(&enode, 0);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
do {
|
|
Jiri Slaby |
256201 |
mt_free_bulk(node->slot_len, slots);
|
|
Jiri Slaby |
256201 |
offset = node->parent_slot + 1;
|
|
Jiri Slaby |
256201 |
- mas.node = node->piv_parent;
|
|
Jiri Slaby |
256201 |
- if (mas_mn(&mas) == node)
|
|
Jiri Slaby |
256201 |
- goto start_slots_free;
|
|
Jiri Slaby |
256201 |
-
|
|
Jiri Slaby |
256201 |
- type = mte_node_type(mas.node);
|
|
Jiri Slaby |
256201 |
- slots = ma_slots(mte_to_node(mas.node), type);
|
|
Jiri Slaby |
256201 |
- if ((offset < mt_slots[type]) && (slots[offset]))
|
|
Jiri Slaby |
256201 |
- slots = mas_dead_walk(&mas, offset);
|
|
Jiri Slaby |
256201 |
-
|
|
Jiri Slaby |
256201 |
- node = mas_mn(&mas;;
|
|
Jiri Slaby |
256201 |
+ enode = node->piv_parent;
|
|
Jiri Slaby |
256201 |
+ if (mte_to_node(enode) == node)
|
|
Jiri Slaby |
256201 |
+ goto free_leaf;
|
|
Jiri Slaby |
256201 |
+
|
|
Jiri Slaby |
256201 |
+ type = mte_node_type(enode);
|
|
Jiri Slaby |
256201 |
+ slots = ma_slots(mte_to_node(enode), type);
|
|
Jiri Slaby |
256201 |
+ if ((offset < mt_slots[type]) &&
|
|
Jiri Slaby |
256201 |
+ rcu_dereference_protected(slots[offset],
|
|
Jiri Slaby |
256201 |
+ lock_is_held(&rcu_callback_map)))
|
|
Jiri Slaby |
256201 |
+ slots = mte_dead_walk(&enode, offset);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
} while ((node != start) || (node->slot_len < offset));
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
slots = ma_slots(node, node->type);
|
|
Jiri Slaby |
256201 |
mt_free_bulk(node->slot_len, slots);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
-start_slots_free:
|
|
Jiri Slaby |
256201 |
- mas_unlock(&mas;;
|
|
Jiri Slaby |
256201 |
free_leaf:
|
|
Jiri Slaby |
256201 |
mt_free_rcu(&node->rcu);
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
-static inline void __rcu **mas_destroy_descend(struct ma_state *mas,
|
|
Jiri Slaby |
256201 |
- struct maple_enode *prev, unsigned char offset)
|
|
Jiri Slaby |
256201 |
+static inline void __rcu **mte_destroy_descend(struct maple_enode **enode,
|
|
Jiri Slaby |
256201 |
+ struct maple_tree *mt, struct maple_enode *prev, unsigned char offset)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
struct maple_node *node;
|
|
Jiri Slaby |
256201 |
- struct maple_enode *next = mas->node;
|
|
Jiri Slaby |
256201 |
+ struct maple_enode *next = *enode;
|
|
Jiri Slaby |
256201 |
void __rcu **slots = NULL;
|
|
Jiri Slaby |
256201 |
+ enum maple_type type;
|
|
Jiri Slaby |
256201 |
+ unsigned char next_offset = 0;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
do {
|
|
Jiri Slaby |
256201 |
- mas->node = next;
|
|
Jiri Slaby |
256201 |
- node = mas_mn(mas);
|
|
Jiri Slaby |
256201 |
- slots = ma_slots(node, mte_node_type(mas->node));
|
|
Jiri Slaby |
256201 |
- next = mas_slot_locked(mas, slots, 0);
|
|
Jiri Slaby |
256201 |
- if ((mte_dead_node(next))) {
|
|
Jiri Slaby |
256201 |
- mte_to_node(next)->type = mte_node_type(next);
|
|
Jiri Slaby |
256201 |
- next = mas_slot_locked(mas, slots, 1);
|
|
Jiri Slaby |
256201 |
- }
|
|
Jiri Slaby |
256201 |
+ *enode = next;
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(*enode);
|
|
Jiri Slaby |
256201 |
+ type = mte_node_type(*enode);
|
|
Jiri Slaby |
256201 |
+ slots = ma_slots(node, type);
|
|
Jiri Slaby |
256201 |
+ next = mt_slot_locked(mt, slots, next_offset);
|
|
Jiri Slaby |
256201 |
+ if ((mte_dead_node(next)))
|
|
Jiri Slaby |
256201 |
+ next = mt_slot_locked(mt, slots, ++next_offset);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- mte_set_node_dead(mas->node);
|
|
Jiri Slaby |
256201 |
- node->type = mte_node_type(mas->node);
|
|
Jiri Slaby |
256201 |
- mas_clear_meta(mas, node, node->type);
|
|
Jiri Slaby |
256201 |
+ mte_set_node_dead(*enode);
|
|
Jiri Slaby |
256201 |
+ node->type = type;
|
|
Jiri Slaby |
256201 |
node->piv_parent = prev;
|
|
Jiri Slaby |
256201 |
node->parent_slot = offset;
|
|
Jiri Slaby |
256201 |
- offset = 0;
|
|
Jiri Slaby |
256201 |
- prev = mas->node;
|
|
Jiri Slaby |
256201 |
+ offset = next_offset;
|
|
Jiri Slaby |
256201 |
+ next_offset = 0;
|
|
Jiri Slaby |
256201 |
+ prev = *enode;
|
|
Jiri Slaby |
256201 |
} while (!mte_is_leaf(next));
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
return slots;
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
-static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags,
|
|
Jiri Slaby |
256201 |
+static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt,
|
|
Jiri Slaby |
256201 |
bool free)
|
|
Jiri Slaby |
256201 |
{
|
|
Jiri Slaby |
256201 |
void __rcu **slots;
|
|
Jiri Slaby |
256201 |
struct maple_node *node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
struct maple_enode *start;
|
|
Jiri Slaby |
256201 |
- struct maple_tree mt;
|
|
Jiri Slaby |
256201 |
-
|
|
Jiri Slaby |
256201 |
- MA_STATE(mas, &mt, 0, 0);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- mas.node = enode;
|
|
Jiri Slaby |
256201 |
if (mte_is_leaf(enode)) {
|
|
Jiri Slaby |
256201 |
node->type = mte_node_type(enode);
|
|
Jiri Slaby |
256201 |
goto free_leaf;
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- ma_flags &= ~MT_FLAGS_LOCK_MASK;
|
|
Jiri Slaby |
256201 |
- mt_init_flags(&mt, ma_flags);
|
|
Jiri Slaby |
256201 |
- mas_lock(&mas;;
|
|
Jiri Slaby |
256201 |
-
|
|
Jiri Slaby |
256201 |
- mte_to_node(enode)->ma_flags = ma_flags;
|
|
Jiri Slaby |
256201 |
start = enode;
|
|
Jiri Slaby |
256201 |
- slots = mas_destroy_descend(&mas, start, 0);
|
|
Jiri Slaby |
256201 |
- node = mas_mn(&mas;;
|
|
Jiri Slaby |
256201 |
+ slots = mte_destroy_descend(&enode, mt, start, 0);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(enode); // Updated in the above call.
|
|
Jiri Slaby |
256201 |
do {
|
|
Jiri Slaby |
256201 |
enum maple_type type;
|
|
Jiri Slaby |
256201 |
unsigned char offset;
|
|
Jiri Slaby |
256201 |
struct maple_enode *parent, *tmp;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- node->type = mte_node_type(mas.node);
|
|
Jiri Slaby |
256201 |
- node->slot_len = mas_dead_leaves(&mas, slots, node->type);
|
|
Jiri Slaby |
256201 |
+ node->slot_len = mte_dead_leaves(enode, mt, slots);
|
|
Jiri Slaby |
256201 |
if (free)
|
|
Jiri Slaby |
256201 |
mt_free_bulk(node->slot_len, slots);
|
|
Jiri Slaby |
256201 |
offset = node->parent_slot + 1;
|
|
Jiri Slaby |
256201 |
- mas.node = node->piv_parent;
|
|
Jiri Slaby |
256201 |
- if (mas_mn(&mas) == node)
|
|
Jiri Slaby |
256201 |
- goto start_slots_free;
|
|
Jiri Slaby |
256201 |
+ enode = node->piv_parent;
|
|
Jiri Slaby |
256201 |
+ if (mte_to_node(enode) == node)
|
|
Jiri Slaby |
256201 |
+ goto free_leaf;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- type = mte_node_type(mas.node);
|
|
Jiri Slaby |
256201 |
- slots = ma_slots(mte_to_node(mas.node), type);
|
|
Jiri Slaby |
256201 |
+ type = mte_node_type(enode);
|
|
Jiri Slaby |
256201 |
+ slots = ma_slots(mte_to_node(enode), type);
|
|
Jiri Slaby |
256201 |
if (offset >= mt_slots[type])
|
|
Jiri Slaby |
256201 |
goto next;
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- tmp = mas_slot_locked(&mas, slots, offset);
|
|
Jiri Slaby |
256201 |
+ tmp = mt_slot_locked(mt, slots, offset);
|
|
Jiri Slaby |
256201 |
if (mte_node_type(tmp) && mte_to_node(tmp)) {
|
|
Jiri Slaby |
256201 |
- parent = mas.node;
|
|
Jiri Slaby |
256201 |
- mas.node = tmp;
|
|
Jiri Slaby |
256201 |
- slots = mas_destroy_descend(&mas, parent, offset);
|
|
Jiri Slaby |
256201 |
+ parent = enode;
|
|
Jiri Slaby |
256201 |
+ enode = tmp;
|
|
Jiri Slaby |
256201 |
+ slots = mte_destroy_descend(&enode, mt, parent, offset);
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
next:
|
|
Jiri Slaby |
256201 |
- node = mas_mn(&mas;;
|
|
Jiri Slaby |
256201 |
- } while (start != mas.node);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
+ } while (start != enode);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
- node = mas_mn(&mas;;
|
|
Jiri Slaby |
256201 |
- node->type = mte_node_type(mas.node);
|
|
Jiri Slaby |
256201 |
- node->slot_len = mas_dead_leaves(&mas, slots, node->type);
|
|
Jiri Slaby |
256201 |
+ node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
+ node->slot_len = mte_dead_leaves(enode, mt, slots);
|
|
Jiri Slaby |
256201 |
if (free)
|
|
Jiri Slaby |
256201 |
mt_free_bulk(node->slot_len, slots);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
-start_slots_free:
|
|
Jiri Slaby |
256201 |
- mas_unlock(&mas;;
|
|
Jiri Slaby |
256201 |
-
|
|
Jiri Slaby |
256201 |
free_leaf:
|
|
Jiri Slaby |
256201 |
if (free)
|
|
Jiri Slaby |
256201 |
mt_free_rcu(&node->rcu);
|
|
Jiri Slaby |
256201 |
else
|
|
Jiri Slaby |
256201 |
- mas_clear_meta(&mas, node, node->type);
|
|
Jiri Slaby |
256201 |
+ mt_clear_meta(mt, node, node->type);
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
/*
|
|
Jiri Slaby |
256201 |
@@ -5682,10 +5686,10 @@ static inline void mte_destroy_walk(struct maple_enode *enode,
|
|
Jiri Slaby |
256201 |
struct maple_node *node = mte_to_node(enode);
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
if (mt_in_rcu(mt)) {
|
|
Jiri Slaby |
256201 |
- mt_destroy_walk(enode, mt->ma_flags, false);
|
|
Jiri Slaby |
256201 |
+ mt_destroy_walk(enode, mt, false);
|
|
Jiri Slaby |
256201 |
call_rcu(&node->rcu, mt_free_walk);
|
|
Jiri Slaby |
256201 |
} else {
|
|
Jiri Slaby |
256201 |
- mt_destroy_walk(enode, mt->ma_flags, true);
|
|
Jiri Slaby |
256201 |
+ mt_destroy_walk(enode, mt, true);
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
}
|
|
Jiri Slaby |
256201 |
|
|
Jiri Slaby |
256201 |
--
|
|
Jiri Slaby |
256201 |
2.35.3
|
|
Jiri Slaby |
256201 |
|