Each unique kernel context, in dept's point of view, should be identified on every entrance to kernel mode e.g. system call or user oriented fault. Otherwise, dept may track meaningless dependencies across different kernel context.
Signed-off-by: Byungchul Park byungchul@sk.com --- include/linux/dept.h | 17 +++++--- include/linux/irq-entry-common.h | 4 ++ include/linux/sched.h | 10 ++--- kernel/dependency/dept.c | 67 ++++++++++++++++--------------- kernel/dependency/dept_internal.h | 12 +++--- 5 files changed, 61 insertions(+), 49 deletions(-)
diff --git a/include/linux/dept.h b/include/linux/dept.h index a6d14b585a8d..2343d8c392d7 100644 --- a/include/linux/dept.h +++ b/include/linux/dept.h @@ -24,11 +24,16 @@ struct task_struct; #define DEPT_MAX_SUBCLASSES_USR (DEPT_MAX_SUBCLASSES / DEPT_MAX_SUBCLASSES_EVT) #define DEPT_MAX_SUBCLASSES_CACHE 2
-#define DEPT_SIRQ 0 -#define DEPT_HIRQ 1 -#define DEPT_IRQS_NR 2 -#define DEPT_SIRQF (1UL << DEPT_SIRQ) -#define DEPT_HIRQF (1UL << DEPT_HIRQ) +enum { + DEPT_CXT_SIRQ = 0, + DEPT_CXT_HIRQ, + DEPT_CXT_IRQS_NR, + DEPT_CXT_PROCESS = DEPT_CXT_IRQS_NR, + DEPT_CXTS_NR +}; + +#define DEPT_SIRQF (1UL << DEPT_CXT_SIRQ) +#define DEPT_HIRQF (1UL << DEPT_CXT_HIRQ)
struct dept_key { union { @@ -146,6 +151,7 @@ extern void dept_event(struct dept_map *m, unsigned long e_f, unsigned long ip, extern void dept_ecxt_exit(struct dept_map *m, unsigned long e_f, unsigned long ip); extern void dept_sched_enter(void); extern void dept_sched_exit(void); +extern void dept_update_cxt(void);
static inline void dept_ecxt_enter_nokeep(struct dept_map *m) { @@ -193,6 +199,7 @@ struct dept_map { }; #define dept_ecxt_exit(m, e_f, ip) do { } while (0) #define dept_sched_enter() do { } while (0) #define dept_sched_exit() do { } while (0) +#define dept_update_cxt() do { } while (0) #define dept_ecxt_enter_nokeep(m) do { } while (0) #define dept_key_init(k) do { (void)(k); } while (0) #define dept_key_destroy(k) do { (void)(k); } while (0) diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 6ab913e57da0..126bac5806a3 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -9,6 +9,7 @@ #include <linux/syscalls.h> #include <linux/tick.h> #include <linux/unwind_deferred.h> +#include <linux/dept.h>
#include <asm/entry-common.h>
@@ -88,6 +89,9 @@ static __always_inline bool arch_in_rcu_eqs(void) { return false; } */ static __always_inline void enter_from_user_mode(struct pt_regs *regs) { + /* Make dept work with a new context. */ + dept_update_cxt(); + arch_enter_from_user_mode(regs); lockdep_hardirqs_off(CALLER_ADDR0);
diff --git a/include/linux/sched.h b/include/linux/sched.h index 33e713c3ba70..1cd0e78f3323 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -833,19 +833,19 @@ struct dept_task { int wait_hist_pos;
/* - * sequential id to identify each IRQ context + * sequential id to identify each context */ - unsigned int irq_id[DEPT_IRQS_NR]; + unsigned int cxt_id[DEPT_CXTS_NR];
/* * for tracking IRQ-enabled points with cross-event */ - unsigned int wgen_enirq[DEPT_IRQS_NR]; + unsigned int wgen_enirq[DEPT_CXT_IRQS_NR];
/* * for keeping up-to-date IRQ-enabled points */ - unsigned long enirq_ip[DEPT_IRQS_NR]; + unsigned long enirq_ip[DEPT_CXT_IRQS_NR];
/* * for reserving a current stack instance at each operation @@ -899,7 +899,7 @@ struct dept_task { .wait_hist = { { .wait = NULL, } }, \ .ecxt_held_pos = 0, \ .wait_hist_pos = 0, \ - .irq_id = { 0U }, \ + .cxt_id = { 0U }, \ .wgen_enirq = { 0U }, \ .enirq_ip = { 0UL }, \ .stack = NULL, \ diff --git a/kernel/dependency/dept.c b/kernel/dependency/dept.c index dfe9dfdb6991..953e1b81a81f 100644 --- a/kernel/dependency/dept.c +++ b/kernel/dependency/dept.c @@ -230,9 +230,9 @@ static struct dept_class *dep_tc(struct dept_dep *d)
static const char *irq_str(int irq) { - if (irq == DEPT_SIRQ) + if (irq == DEPT_CXT_SIRQ) return "softirq"; - if (irq == DEPT_HIRQ) + if (irq == DEPT_CXT_HIRQ) return "hardirq"; return "(unknown)"; } @@ -410,7 +410,7 @@ static void initialize_class(struct dept_class *c) { int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { struct dept_iecxt *ie = &c->iecxt[i]; struct dept_iwait *iw = &c->iwait[i];
@@ -436,7 +436,7 @@ static void initialize_ecxt(struct dept_ecxt *e) { int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { e->enirq_stack[i] = NULL; e->enirq_ip[i] = 0UL; } @@ -452,7 +452,7 @@ static void initialize_wait(struct dept_wait *w) { int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { w->irq_stack[i] = NULL; w->irq_ip[i] = 0UL; } @@ -491,7 +491,7 @@ static void destroy_ecxt(struct dept_ecxt *e) { int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) if (e->enirq_stack[i]) put_stack(e->enirq_stack[i]); if (e->class) @@ -507,7 +507,7 @@ static void destroy_wait(struct dept_wait *w) { int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) if (w->irq_stack[i]) put_stack(w->irq_stack[i]); if (w->class) @@ -665,7 +665,7 @@ static void print_diagram(struct dept_dep *d) const char *tc_n = tc->sched_map ? "<sched>" : (tc->name ?: "(unknown)");
irqf = e->enirqf & w->irqf; - for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) { + for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) { if (!firstline) pr_warn("\nor\n\n"); firstline = false; @@ -698,7 +698,7 @@ static void print_dep(struct dept_dep *d) const char *tc_n = tc->sched_map ? "<sched>" : (tc->name ?: "(unknown)");
irqf = e->enirqf & w->irqf; - for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) { + for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) { pr_warn("%s has been enabled:\n", irq_str(irq)); print_ip_stack(e->enirq_ip[irq], e->enirq_stack[irq]); pr_warn("\n"); @@ -866,7 +866,7 @@ static void bfs(void *root, struct bfs_ops *ops, void *in, void **out) */
static unsigned long cur_enirqf(void); -static int cur_irq(void); +static int cur_cxt(void); static unsigned int cur_ctxt_id(void);
static struct dept_iecxt *iecxt(struct dept_class *c, int irq) @@ -1443,7 +1443,7 @@ static void add_dep(struct dept_ecxt *e, struct dept_wait *w) if (d) { check_dl_bfs(d);
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { struct dept_iwait *fiw = iwait(fc, i); struct dept_iecxt *found_ie; struct dept_iwait *found_iw; @@ -1487,7 +1487,7 @@ static void add_wait(struct dept_class *c, unsigned long ip, struct dept_task *dt = dept_task(); struct dept_wait *w; unsigned int wg; - int irq; + int cxt; int i;
if (DEPT_WARN_ON(!valid_class(c))) @@ -1503,9 +1503,9 @@ static void add_wait(struct dept_class *c, unsigned long ip, w->wait_stack = get_current_stack(); w->sched_sleep = sched_sleep;
- irq = cur_irq(); - if (irq < DEPT_IRQS_NR) - add_iwait(c, irq, w); + cxt = cur_cxt(); + if (cxt == DEPT_CXT_HIRQ || cxt == DEPT_CXT_SIRQ) + add_iwait(c, cxt, w);
/* * Avoid adding dependency between user aware nested ecxt and @@ -1579,7 +1579,7 @@ static struct dept_ecxt_held *add_ecxt(struct dept_map *m, eh->sub_l = sub_l;
irqf = cur_enirqf(); - for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) + for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) add_iecxt(c, irq, e, false);
del_ecxt(e); @@ -1728,7 +1728,7 @@ static void do_event(struct dept_map *m, struct dept_map *real_m, add_dep(eh->ecxt, wh->wait); }
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { struct dept_ecxt *e;
if (before(dt->wgen_enirq[i], wg)) @@ -1775,7 +1775,7 @@ static void disconnect_class(struct dept_class *c) call_rcu(&d->rh, del_dep_rcu); }
- for (i = 0; i < DEPT_IRQS_NR; i++) { + for (i = 0; i < DEPT_CXT_IRQS_NR; i++) { stale_iecxt(iecxt(c, i)); stale_iwait(iwait(c, i)); } @@ -1800,27 +1800,21 @@ static unsigned long cur_enirqf(void) return 0UL; }
-static int cur_irq(void) +static int cur_cxt(void) { if (lockdep_softirq_context(current)) - return DEPT_SIRQ; + return DEPT_CXT_SIRQ; if (lockdep_hardirq_context()) - return DEPT_HIRQ; - return DEPT_IRQS_NR; + return DEPT_CXT_HIRQ; + return DEPT_CXT_PROCESS; }
static unsigned int cur_ctxt_id(void) { struct dept_task *dt = dept_task(); - int irq = cur_irq(); + int cxt = cur_cxt();
- /* - * Normal process context - */ - if (irq == DEPT_IRQS_NR) - return 0U; - - return dt->irq_id[irq] | (1UL << irq); + return dt->cxt_id[cxt] | (1UL << cxt); }
static void enirq_transition(int irq) @@ -1877,7 +1871,7 @@ static void dept_enirq(unsigned long ip)
flags = dept_enter();
- for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) { + for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) { dt->enirq_ip[irq] = ip; enirq_transition(irq); } @@ -1923,6 +1917,13 @@ void noinstr dept_hardirqs_off(void) dept_task()->hardirqs_enabled = false; }
+void noinstr dept_update_cxt(void) +{ + struct dept_task *dt = dept_task(); + + dt->cxt_id[DEPT_CXT_PROCESS] += 1UL << DEPT_CXTS_NR; +} + /* * Ensure it's the outmost softirq context. */ @@ -1930,7 +1931,7 @@ void dept_softirq_enter(void) { struct dept_task *dt = dept_task();
- dt->irq_id[DEPT_SIRQ] += 1UL << DEPT_IRQS_NR; + dt->cxt_id[DEPT_CXT_SIRQ] += 1UL << DEPT_CXTS_NR; }
/* @@ -1940,7 +1941,7 @@ void noinstr dept_hardirq_enter(void) { struct dept_task *dt = dept_task();
- dt->irq_id[DEPT_HIRQ] += 1UL << DEPT_IRQS_NR; + dt->cxt_id[DEPT_CXT_HIRQ] += 1UL << DEPT_CXTS_NR; }
void dept_sched_enter(void) diff --git a/kernel/dependency/dept_internal.h b/kernel/dependency/dept_internal.h index a1eb1ed647a7..262114a0110c 100644 --- a/kernel/dependency/dept_internal.h +++ b/kernel/dependency/dept_internal.h @@ -106,8 +106,8 @@ struct dept_class { /* * for tracking IRQ dependencies */ - struct dept_iecxt iecxt[DEPT_IRQS_NR]; - struct dept_iwait iwait[DEPT_IRQS_NR]; + struct dept_iecxt iecxt[DEPT_CXT_IRQS_NR]; + struct dept_iwait iwait[DEPT_CXT_IRQS_NR];
/* * classified by a map embedded in task_struct, @@ -169,8 +169,8 @@ struct dept_ecxt { /* * where the IRQ-enabled happened */ - unsigned long enirq_ip[DEPT_IRQS_NR]; - struct dept_stack *enirq_stack[DEPT_IRQS_NR]; + unsigned long enirq_ip[DEPT_CXT_IRQS_NR]; + struct dept_stack *enirq_stack[DEPT_CXT_IRQS_NR];
/* * where the event context started @@ -214,8 +214,8 @@ struct dept_wait { /* * where the IRQ wait happened */ - unsigned long irq_ip[DEPT_IRQS_NR]; - struct dept_stack *irq_stack[DEPT_IRQS_NR]; + unsigned long irq_ip[DEPT_CXT_IRQS_NR]; + struct dept_stack *irq_stack[DEPT_CXT_IRQS_NR];
/* * where the wait happened