|
Jiri Wiesner |
37a830 |
From 1d61659ced6bd8881cf2fb5cbcb28f9541fc7430 Mon Sep 17 00:00:00 2001
|
|
Jiri Wiesner |
aa6acc |
From: Waiman Long <longman@redhat.com>
|
|
Jiri Wiesner |
37a830 |
Date: Wed, 25 Jan 2023 19:36:27 -0500
|
|
Jiri Wiesner |
37a830 |
Subject: [PATCH] locking/rwsem: Disable preemption in all down_write*() and
|
|
Jiri Wiesner |
aa6acc |
up_write() code paths
|
|
Jiri Wiesner |
37a830 |
Git-commit: 1d61659ced6bd8881cf2fb5cbcb28f9541fc7430
|
|
Jiri Wiesner |
37a830 |
Patch-mainline: v6.3-rc1
|
|
Jiri Wiesner |
37a830 |
References: bsc#1204996
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
37a830 |
The previous patch has disabled preemption in all the down_read() and
|
|
Jiri Wiesner |
37a830 |
up_read() code paths. For symmetry, this patch extends commit:
|
|
Jiri Wiesner |
37a830 |
|
|
Jiri Wiesner |
37a830 |
48dfb5d2560d ("locking/rwsem: Disable preemption while trying for rwsem lock")
|
|
Jiri Wiesner |
37a830 |
|
|
Jiri Wiesner |
37a830 |
... to have preemption disabled in all the down_write() and up_write()
|
|
Jiri Wiesner |
37a830 |
code paths, including downgrade_write().
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
Suggested-by: Peter Zijlstra <peterz@infradead.org>
|
|
Jiri Wiesner |
aa6acc |
Signed-off-by: Waiman Long <longman@redhat.com>
|
|
Jiri Wiesner |
37a830 |
Signed-off-by: Ingo Molnar <mingo@kernel.org>
|
|
Jiri Wiesner |
37a830 |
Link: https://lore.kernel.org/r/20230126003628.365092-4-longman@redhat.com
|
|
Jiri Wiesner |
aa6acc |
Signed-off-by: Jiri Wiesner <jwiesner@suse.de>
|
|
Jiri Wiesner |
aa6acc |
---
|
|
Jiri Wiesner |
aa6acc |
kernel/locking/rwsem.c | 38 +++++++++++++++++++-------------------
|
|
Jiri Wiesner |
aa6acc |
1 file changed, 19 insertions(+), 19 deletions(-)
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
|
|
Jiri Wiesner |
37a830 |
index 84d5b649b95f..acb5a50309a1 100644
|
|
Jiri Wiesner |
aa6acc |
--- a/kernel/locking/rwsem.c
|
|
Jiri Wiesner |
aa6acc |
+++ b/kernel/locking/rwsem.c
|
|
Jiri Wiesner |
37a830 |
@@ -256,16 +256,13 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)
|
|
Jiri Wiesner |
aa6acc |
static inline bool rwsem_write_trylock(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
{
|
|
Jiri Wiesner |
aa6acc |
long tmp = RWSEM_UNLOCKED_VALUE;
|
|
Jiri Wiesner |
aa6acc |
- bool ret = false;
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {
|
|
Jiri Wiesner |
aa6acc |
rwsem_set_owner(sem);
|
|
Jiri Wiesner |
aa6acc |
- ret = true;
|
|
Jiri Wiesner |
aa6acc |
+ return true;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
- return ret;
|
|
Jiri Wiesner |
aa6acc |
+ return false;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
/*
|
|
Jiri Wiesner |
37a830 |
@@ -716,7 +713,6 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
return false;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
/*
|
|
Jiri Wiesner |
aa6acc |
* Disable preemption is equal to the RCU read-side crital section,
|
|
Jiri Wiesner |
aa6acc |
* thus the task_strcut structure won't go away.
|
|
Jiri Wiesner |
37a830 |
@@ -728,7 +724,6 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
if ((flags & RWSEM_NONSPINNABLE) ||
|
|
Jiri Wiesner |
aa6acc |
(owner && !(flags & RWSEM_READER_OWNED) && !owner_on_cpu(owner)))
|
|
Jiri Wiesner |
aa6acc |
ret = false;
|
|
Jiri Wiesner |
aa6acc |
- preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
lockevent_cond_inc(rwsem_opt_fail, !ret);
|
|
Jiri Wiesner |
aa6acc |
return ret;
|
|
Jiri Wiesner |
37a830 |
@@ -828,8 +823,6 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
int loop = 0;
|
|
Jiri Wiesner |
aa6acc |
u64 rspin_threshold = 0;
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
-
|
|
Jiri Wiesner |
aa6acc |
/* sem->wait_lock should not be held when doing optimistic spinning */
|
|
Jiri Wiesner |
aa6acc |
if (!osq_lock(&sem->osq))
|
|
Jiri Wiesner |
aa6acc |
goto done;
|
|
Jiri Wiesner |
37a830 |
@@ -937,7 +930,6 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
osq_unlock(&sem->osq);
|
|
Jiri Wiesner |
aa6acc |
done:
|
|
Jiri Wiesner |
aa6acc |
- preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
lockevent_cond_inc(rwsem_opt_fail, !taken);
|
|
Jiri Wiesner |
aa6acc |
return taken;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
37a830 |
@@ -1178,15 +1170,12 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
|
|
Jiri Wiesner |
aa6acc |
if (waiter.handoff_set) {
|
|
Jiri Wiesner |
aa6acc |
enum owner_state owner_state;
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
owner_state = rwsem_spin_on_owner(sem);
|
|
Jiri Wiesner |
aa6acc |
- preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
-
|
|
Jiri Wiesner |
aa6acc |
if (owner_state == OWNER_NULL)
|
|
Jiri Wiesner |
aa6acc |
goto trylock_again;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
- schedule();
|
|
Jiri Wiesner |
aa6acc |
+ schedule_preempt_disabled();
|
|
Jiri Wiesner |
aa6acc |
lockevent_inc(rwsem_sleep_writer);
|
|
Jiri Wiesner |
aa6acc |
set_current_state(state);
|
|
Jiri Wiesner |
aa6acc |
trylock_again:
|
|
Jiri Wiesner |
37a830 |
@@ -1310,12 +1299,15 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
*/
|
|
Jiri Wiesner |
aa6acc |
static inline int __down_write_common(struct rw_semaphore *sem, int state)
|
|
Jiri Wiesner |
aa6acc |
{
|
|
Jiri Wiesner |
aa6acc |
+ int ret = 0;
|
|
Jiri Wiesner |
aa6acc |
+
|
|
Jiri Wiesner |
aa6acc |
+ preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
if (unlikely(!rwsem_write_trylock(sem))) {
|
|
Jiri Wiesner |
aa6acc |
if (IS_ERR(rwsem_down_write_slowpath(sem, state)))
|
|
Jiri Wiesner |
aa6acc |
- return -EINTR;
|
|
Jiri Wiesner |
aa6acc |
+ ret = -EINTR;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
-
|
|
Jiri Wiesner |
aa6acc |
- return 0;
|
|
Jiri Wiesner |
aa6acc |
+ preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
+ return ret;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
static inline void __down_write(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
37a830 |
@@ -1330,8 +1322,14 @@ static inline int __down_write_killable(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
static inline int __down_write_trylock(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
{
|
|
Jiri Wiesner |
aa6acc |
+ int ret;
|
|
Jiri Wiesner |
aa6acc |
+
|
|
Jiri Wiesner |
aa6acc |
+ preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
|
|
Jiri Wiesner |
aa6acc |
- return rwsem_write_trylock(sem);
|
|
Jiri Wiesner |
aa6acc |
+ ret = rwsem_write_trylock(sem);
|
|
Jiri Wiesner |
aa6acc |
+ preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
+
|
|
Jiri Wiesner |
aa6acc |
+ return ret;
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
/*
|
|
Jiri Wiesner |
37a830 |
@@ -1374,9 +1372,9 @@ static inline void __up_write(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
rwsem_clear_owner(sem);
|
|
Jiri Wiesner |
aa6acc |
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
|
|
Jiri Wiesner |
aa6acc |
- preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
if (unlikely(tmp & RWSEM_FLAG_WAITERS))
|
|
Jiri Wiesner |
aa6acc |
rwsem_wake(sem);
|
|
Jiri Wiesner |
aa6acc |
+ preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
/*
|
|
Jiri Wiesner |
37a830 |
@@ -1394,11 +1392,13 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
|
|
Jiri Wiesner |
aa6acc |
* write side. As such, rely on RELEASE semantics.
|
|
Jiri Wiesner |
aa6acc |
*/
|
|
Jiri Wiesner |
aa6acc |
DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem);
|
|
Jiri Wiesner |
aa6acc |
+ preempt_disable();
|
|
Jiri Wiesner |
aa6acc |
tmp = atomic_long_fetch_add_release(
|
|
Jiri Wiesner |
aa6acc |
-RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count);
|
|
Jiri Wiesner |
aa6acc |
rwsem_set_reader_owned(sem);
|
|
Jiri Wiesner |
aa6acc |
if (tmp & RWSEM_FLAG_WAITERS)
|
|
Jiri Wiesner |
aa6acc |
rwsem_downgrade_wake(sem);
|
|
Jiri Wiesner |
aa6acc |
+ preempt_enable();
|
|
Jiri Wiesner |
aa6acc |
}
|
|
Jiri Wiesner |
aa6acc |
|
|
Jiri Wiesner |
aa6acc |
#else /* !CONFIG_PREEMPT_RT */
|
|
Jiri Wiesner |
aa6acc |
--
|
|
Jiri Wiesner |
aa6acc |
2.35.3
|
|
Jiri Wiesner |
aa6acc |
|