How to place dept this way looks so ugly. However, it's inevitable for now. The way should be enhanced gradually.
Signed-off-by: Byungchul Park byungchul@sk.com --- include/linux/irqflags.h | 7 +- include/linux/local_lock_internal.h | 1 + include/linux/lockdep.h | 102 ++++++++++++++++++++++------ include/linux/lockdep_types.h | 3 + include/linux/mutex.h | 1 + include/linux/percpu-rwsem.h | 2 +- include/linux/rtmutex.h | 1 + include/linux/rwlock_types.h | 1 + include/linux/rwsem.h | 1 + include/linux/seqlock.h | 2 +- include/linux/spinlock_types_raw.h | 3 + include/linux/srcu.h | 2 +- kernel/dependency/dept.c | 8 +-- kernel/locking/lockdep.c | 22 ++++++ 14 files changed, 127 insertions(+), 29 deletions(-)
diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index 57b074e0cfbb..d8b9cf093f83 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -15,6 +15,7 @@ #include <linux/irqflags_types.h> #include <linux/typecheck.h> #include <linux/cleanup.h> +#include <linux/dept.h> #include <asm/irqflags.h> #include <asm/percpu.h>
@@ -55,8 +56,10 @@ extern void trace_hardirqs_off(void); # define lockdep_softirqs_enabled(p) ((p)->softirqs_enabled) # define lockdep_hardirq_enter() \ do { \ - if (__this_cpu_inc_return(hardirq_context) == 1)\ + if (__this_cpu_inc_return(hardirq_context) == 1) { \ current->hardirq_threaded = 0; \ + dept_hardirq_enter(); \ + } \ } while (0) # define lockdep_hardirq_threaded() \ do { \ @@ -131,6 +134,8 @@ do { \ # define lockdep_softirq_enter() \ do { \ current->softirq_context++; \ + if (current->softirq_context == 1) \ + dept_softirq_enter(); \ } while (0) # define lockdep_softirq_exit() \ do { \ diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 234be7f12c15..09255c5a665f 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -35,6 +35,7 @@ typedef struct local_trylock local_trylock_t; .name = #lockname, \ .wait_type_inner = LD_WAIT_CONFIG, \ .lock_type = LD_LOCK_PERCPU, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ }, \ .owner = NULL,
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..60b17a2ff555 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -12,6 +12,7 @@
#include <linux/lockdep_types.h> #include <linux/smp.h> +#include <linux/dept_ldt.h> #include <asm/percpu.h>
struct task_struct; @@ -39,6 +40,8 @@ static inline void lockdep_copy_map(struct lockdep_map *to, */ for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) to->class_cache[i] = NULL; + + dept_map_copy(&to->dmap, &from->dmap); }
/* @@ -428,7 +431,8 @@ enum xhlock_context_t { * Note that _name must not be NULL. */ #define STATIC_LOCKDEP_MAP_INIT(_name, _key) \ - { .name = (_name), .key = (void *)(_key), } + { .name = (_name), .key = (void *)(_key), \ + .dmap = DEPT_MAP_INITIALIZER(_name, _key) }
static inline void lockdep_invariant_state(bool force) {} static inline void lockdep_free_task(struct task_struct *task) {} @@ -510,33 +514,89 @@ extern bool read_lock_is_recursive(void); #define lock_acquire_shared(l, s, t, n, i) lock_acquire(l, s, t, 1, 1, n, i) #define lock_acquire_shared_recursive(l, s, t, n, i) lock_acquire(l, s, t, 2, 1, n, i)
-#define spin_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) -#define spin_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) -#define spin_release(l, i) lock_release(l, i) - -#define rwlock_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) +#define spin_acquire(l, s, t, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_exclusive(l, s, t, NULL, i); \ +} while (0) +#define spin_acquire_nest(l, s, t, n, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, n, i); \ + lock_acquire_exclusive(l, s, t, n, i); \ +} while (0) +#define spin_release(l, i) \ +do { \ + ldt_unlock(&(l)->dmap, i); \ + lock_release(l, i); \ +} while (0) +#define rwlock_acquire(l, s, t, i) \ +do { \ + ldt_wlock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_exclusive(l, s, t, NULL, i); \ +} while (0) #define rwlock_acquire_read(l, s, t, i) \ do { \ + ldt_rlock(&(l)->dmap, s, t, NULL, i, !read_lock_is_recursive());\ if (read_lock_is_recursive()) \ lock_acquire_shared_recursive(l, s, t, NULL, i); \ else \ lock_acquire_shared(l, s, t, NULL, i); \ } while (0) - -#define rwlock_release(l, i) lock_release(l, i) - -#define seqcount_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) -#define seqcount_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i) -#define seqcount_release(l, i) lock_release(l, i) - -#define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) -#define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) -#define mutex_release(l, i) lock_release(l, i) - -#define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) -#define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) -#define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i) -#define rwsem_release(l, i) lock_release(l, i) +#define rwlock_release(l, i) \ +do { \ + ldt_unlock(&(l)->dmap, i); \ + lock_release(l, i); \ +} while (0) +#define seqcount_acquire(l, s, t, i) \ +do { \ + ldt_wlock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_exclusive(l, s, t, NULL, i); \ +} while (0) +#define seqcount_acquire_read(l, s, t, i) \ +do { \ + ldt_rlock(&(l)->dmap, s, t, NULL, i, false); \ + lock_acquire_shared_recursive(l, s, t, NULL, i); \ +} while (0) +#define seqcount_release(l, i) \ +do { \ + ldt_unlock(&(l)->dmap, i); \ + lock_release(l, i); \ +} while (0) +#define mutex_acquire(l, s, t, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_exclusive(l, s, t, NULL, i); \ +} while (0) +#define mutex_acquire_nest(l, s, t, n, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, n, i); \ + lock_acquire_exclusive(l, s, t, n, i); \ +} while (0) +#define mutex_release(l, i) \ +do { \ + ldt_unlock(&(l)->dmap, i); \ + lock_release(l, i); \ +} while (0) +#define rwsem_acquire(l, s, t, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_exclusive(l, s, t, NULL, i); \ +} while (0) +#define rwsem_acquire_nest(l, s, t, n, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, n, i); \ + lock_acquire_exclusive(l, s, t, n, i); \ +} while (0) +#define rwsem_acquire_read(l, s, t, i) \ +do { \ + ldt_lock(&(l)->dmap, s, t, NULL, i); \ + lock_acquire_shared(l, s, t, NULL, i); \ +} while (0) +#define rwsem_release(l, i) \ +do { \ + ldt_unlock(&(l)->dmap, i); \ + lock_release(l, i); \ +} while (0)
#define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_) #define lock_map_acquire_try(l) lock_acquire_exclusive(l, 0, 1, NULL, _THIS_IP_) diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..0c3389ed26b6 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -11,6 +11,7 @@ #define __LINUX_LOCKDEP_TYPES_H
#include <linux/types.h> +#include <linux/dept.h>
#define MAX_LOCKDEP_SUBCLASSES 8UL
@@ -77,6 +78,7 @@ struct lock_class_key { struct hlist_node hash_entry; struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES]; }; + struct dept_key dkey; };
extern struct lock_class_key __lockdep_no_validate__; @@ -195,6 +197,7 @@ struct lockdep_map { int cpu; unsigned long ip; #endif + struct dept_map dmap; };
struct pin_cookie { unsigned int val; }; diff --git a/include/linux/mutex.h b/include/linux/mutex.h index ecaa0440f6ec..3d9bc1a28569 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -29,6 +29,7 @@ struct device; , .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_SLEEP, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ } #else # define __DEP_MAP_MUTEX_INITIALIZER(lockname) diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index c8cb010d655e..ca9522f0882b 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -22,7 +22,7 @@ struct percpu_rw_semaphore { };
#ifdef CONFIG_DEBUG_LOCK_ALLOC -#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }, +#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname, .dmap = DEPT_MAP_INITIALIZER(lockname, NULL) }, #else #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) #endif diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index ede4c6bf6f22..ac68c3e5e2ec 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -91,6 +91,7 @@ do { \ .dep_map = { \ .name = #mutexname, \ .wait_type_inner = LD_WAIT_SLEEP, \ + .dmap = DEPT_MAP_INITIALIZER(mutexname, NULL),\ } #else #define __DEP_MAP_RT_MUTEX_INITIALIZER(mutexname) diff --git a/include/linux/rwlock_types.h b/include/linux/rwlock_types.h index d5e7316401e7..f2ff62ef4c36 100644 --- a/include/linux/rwlock_types.h +++ b/include/linux/rwlock_types.h @@ -10,6 +10,7 @@ .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_CONFIG, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL), \ } #else # define RW_DEP_MAP_INIT(lockname) diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index 9bf1d93d3d7b..47ab3fcee48b 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -22,6 +22,7 @@ .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_SLEEP, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ }, #else # define __RWSEM_DEP_MAP_INIT(lockname) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 5a40252b8334..efc93acf1612 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -52,7 +52,7 @@ static inline void __seqcount_init(seqcount_t *s, const char *name, #ifdef CONFIG_DEBUG_LOCK_ALLOC
# define SEQCOUNT_DEP_MAP_INIT(lockname) \ - .dep_map = { .name = #lockname } + .dep_map = { .name = #lockname, .dmap = DEPT_MAP_INITIALIZER(lockname, NULL) }
/** * seqcount_init() - runtime initializer for seqcount_t diff --git a/include/linux/spinlock_types_raw.h b/include/linux/spinlock_types_raw.h index e5644ab2161f..5f245afdd77c 100644 --- a/include/linux/spinlock_types_raw.h +++ b/include/linux/spinlock_types_raw.h @@ -32,11 +32,13 @@ typedef struct raw_spinlock raw_spinlock_t; .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_SPIN, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ } # define SPIN_DEP_MAP_INIT(lockname) \ .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_CONFIG, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ }
# define LOCAL_SPIN_DEP_MAP_INIT(lockname) \ @@ -44,6 +46,7 @@ typedef struct raw_spinlock raw_spinlock_t; .name = #lockname, \ .wait_type_inner = LD_WAIT_CONFIG, \ .lock_type = LD_LOCK_PERCPU, \ + .dmap = DEPT_MAP_INITIALIZER(lockname, NULL),\ } #else # define RAW_SPIN_DEP_MAP_INIT(lockname) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index bb44a0bd7696..50c78f71ad43 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -53,7 +53,7 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ })
-#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, +#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name, .dmap = DEPT_MAP_INITIALIZER(srcu_name, NULL) }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
int init_srcu_struct(struct srcu_struct *ssp); diff --git a/kernel/dependency/dept.c b/kernel/dependency/dept.c index 52b9669fe017..47fef2207af1 100644 --- a/kernel/dependency/dept.c +++ b/kernel/dependency/dept.c @@ -250,10 +250,10 @@ static bool dept_working(void) * Even k == NULL is considered a valid key because it would use * &->map_key as the key in that case. */ -struct dept_key __dept_no_validate__; +extern struct lock_class_key __lockdep_no_validate__; static bool valid_key(struct dept_key *k) { - return &__dept_no_validate__ != k; + return &__lockdep_no_validate__.dkey != k; }
/* @@ -1921,7 +1921,7 @@ void dept_softirqs_off(void) dept_task()->softirqs_enabled = false; }
-void dept_hardirqs_off(void) +void noinstr dept_hardirqs_off(void) { /* * Assumes that it's called with IRQ disabled so that accessing @@ -1943,7 +1943,7 @@ void dept_softirq_enter(void) /* * Ensure it's the outmost hardirq context. */ -void dept_hardirq_enter(void) +void noinstr dept_hardirq_enter(void) { struct dept_task *dt = dept_task();
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..dc97f2753ef8 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -1224,6 +1224,8 @@ void lockdep_register_key(struct lock_class_key *key) struct lock_class_key *k; unsigned long flags;
+ dept_key_init(&key->dkey); + if (WARN_ON_ONCE(static_obj(key))) return; hash_head = keyhashentry(key); @@ -4361,6 +4363,8 @@ static void __trace_hardirqs_on_caller(void) */ void lockdep_hardirqs_on_prepare(void) { + dept_hardirqs_on(); + if (unlikely(!debug_locks)) return;
@@ -4481,6 +4485,8 @@ EXPORT_SYMBOL_GPL(lockdep_hardirqs_on); */ void noinstr lockdep_hardirqs_off(unsigned long ip) { + dept_hardirqs_off(); + if (unlikely(!debug_locks)) return;
@@ -4525,6 +4531,8 @@ void lockdep_softirqs_on(unsigned long ip) { struct irqtrace_events *trace = ¤t->irqtrace;
+ dept_softirqs_on_ip(ip); + if (unlikely(!lockdep_enabled())) return;
@@ -4563,6 +4571,8 @@ void lockdep_softirqs_on(unsigned long ip) */ void lockdep_softirqs_off(unsigned long ip) { + dept_softirqs_off(); + if (unlikely(!lockdep_enabled())) return;
@@ -4940,6 +4950,8 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, { int i;
+ ldt_init(&lock->dmap, &key->dkey, subclass, name); + for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) lock->class_cache[i] = NULL;
@@ -5736,6 +5748,12 @@ void lock_set_class(struct lockdep_map *lock, const char *name, { unsigned long flags;
+ /* + * dept_map_(re)init() might be called twice redundantly. But + * there's no choice as long as Dept relies on Lockdep. + */ + ldt_set_class(&lock->dmap, name, &key->dkey, subclass, ip); + if (unlikely(!lockdep_enabled())) return;
@@ -5753,6 +5771,8 @@ void lock_downgrade(struct lockdep_map *lock, unsigned long ip) { unsigned long flags;
+ ldt_downgrade(&lock->dmap, ip); + if (unlikely(!lockdep_enabled())) return;
@@ -6588,6 +6608,8 @@ void lockdep_unregister_key(struct lock_class_key *key) bool found = false; bool need_callback = false;
+ dept_key_destroy(&key->dkey); + might_sleep();
if (WARN_ON_ONCE(static_obj(key)))