[PATCH v4 02/38] perf: Skip pmu_ctx based on event_type

24 Mar 2025

From: Kan Liang kan.liang@linux.intel.com
To optimize the cgroup context switch, the perf_event_pmu_context
iteration skips the PMUs without cgroup events. A bool cgroup was
introduced to indicate the case. It can work, but this way is hard to
extend for other cases, e.g. skipping non-passthrough PMUs. It doesn't
make sense to keep adding bool variables.
Pass the event_type instead of the specific bool variable. Check both
the event_type and related pmu_ctx variables to decide whether skipping
a PMU.
Event flags, e.g., EVENT_CGROUP, should be cleard in the ctx->is_active.
Add EVENT_FLAGS to indicate such event flags.
No functional change.
Signed-off-by: Kan Liang kan.liang@linux.intel.com
Tested-by: Yongwei Ma yongwei.ma@intel.com
Signed-off-by: Mingwei Zhang mizhang@google.com
---
 kernel/events/core.c | 73 ++++++++++++++++++++++++--------------------
 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/kernel/events/core.c b/kernel/events/core.c
index be623701dc48..8d3a0cc59fb4 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -163,7 +163,7 @@ enum event_type_t {
    /* see ctx_resched() for details */
    EVENT_CPU	= 0x10,
    EVENT_CGROUP	= 0x20,
-
+	EVENT_FLAGS	= EVENT_CGROUP,
    /* compound helpers */
    EVENT_ALL         = EVENT_FLEXIBLE | EVENT_PINNED,
    EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN,
@@ -733,27 +733,37 @@ do {									\
    ___p;								\
 })
-#define for_each_epc(_epc, _ctx, _pmu, _cgroup)				\
+static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx,
+			      enum event_type_t event_type)
+{
+	if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups)
+		return true;
+	return false;
+}
+
+#define for_each_epc(_epc, _ctx, _pmu, _event_type)			\
    list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
-		if (_cgroup && !_epc->nr_cgroups)			\
+		if (perf_skip_pmu_ctx(_epc, _event_type))		\
    		continue;					\
    	else if (_pmu && _epc->pmu != _pmu)			\
    		continue;					\
    	else
-static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
+static void perf_ctx_disable(struct perf_event_context *ctx,
+			     enum event_type_t event_type)
 {
    struct perf_event_pmu_context *pmu_ctx;
-	for_each_epc(pmu_ctx, ctx, NULL, cgroup)
+	for_each_epc(pmu_ctx, ctx, NULL, event_type)
    	perf_pmu_disable(pmu_ctx->pmu);
 }
-static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
+static void perf_ctx_enable(struct perf_event_context *ctx,
+			    enum event_type_t event_type)
 {
    struct perf_event_pmu_context *pmu_ctx;
-	for_each_epc(pmu_ctx, ctx, NULL, cgroup)
+	for_each_epc(pmu_ctx, ctx, NULL, event_type)
    	perf_pmu_enable(pmu_ctx->pmu);
 }
@@ -913,7 +923,7 @@ static void perf_cgroup_switch(struct task_struct *task)
    	return;
perf_ctx_lock(cpuctx, cpuctx->task_ctx);
-	perf_ctx_disable(&cpuctx->ctx, true);
+	perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP);
ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
    /*
@@ -929,7 +939,7 @@ static void perf_cgroup_switch(struct task_struct *task)
     */
    ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
-	perf_ctx_enable(&cpuctx->ctx, true);
+	perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP);
    perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 }
@@ -2796,11 +2806,11 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
event_type &= EVENT_ALL;
-	for_each_epc(epc, &cpuctx->ctx, pmu, false)
+	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
    	perf_pmu_disable(epc->pmu);
if (task_ctx) {
-		for_each_epc(epc, task_ctx, pmu, false)
+		for_each_epc(epc, task_ctx, pmu, 0)
    		perf_pmu_disable(epc->pmu);
task_ctx_sched_out(task_ctx, pmu, event_type);
@@ -2820,11 +2830,11 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
perf_event_sched_in(cpuctx, task_ctx, pmu);
-	for_each_epc(epc, &cpuctx->ctx, pmu, false)
+	for_each_epc(epc, &cpuctx->ctx, pmu, 0)
    	perf_pmu_enable(epc->pmu);
if (task_ctx) {
-		for_each_epc(epc, task_ctx, pmu, false)
+		for_each_epc(epc, task_ctx, pmu, 0)
    		perf_pmu_enable(epc->pmu);
    }
 }
@@ -3374,11 +3384,10 @@ static void
 ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
 {
    struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
    struct perf_event_pmu_context *pmu_ctx;
    int is_active = ctx->is_active;
-	bool cgroup = event_type & EVENT_CGROUP;
-	event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock);
@@ -3409,7 +3418,7 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t
     * see __load_acquire() in perf_event_time_now()
     */
    barrier();
-	ctx->is_active &= ~event_type;
+	ctx->is_active &= ~active_type;
if (!(ctx->is_active & EVENT_ALL)) {
    	/*
@@ -3430,7 +3439,7 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t
is_active ^= ctx->is_active; /* changed bits */
-	for_each_epc(pmu_ctx, ctx, pmu, cgroup)
+	for_each_epc(pmu_ctx, ctx, pmu, event_type)
    	__pmu_ctx_sched_out(pmu_ctx, is_active);
 }
@@ -3622,7 +3631,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
    	raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
    	if (context_equiv(ctx, next_ctx)) {
-			perf_ctx_disable(ctx, false);
+			perf_ctx_disable(ctx, 0);
/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */
    		if (local_read(&ctx->nr_no_switch_fast) ||
@@ -3647,7 +3656,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
    		perf_ctx_sched_task_cb(ctx, false);
    		perf_event_swap_task_ctx_data(ctx, next_ctx);
-			perf_ctx_enable(ctx, false);
+			perf_ctx_enable(ctx, 0);
/*
    		 * RCU_INIT_POINTER here is safe because we've not
@@ -3671,13 +3680,13 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
if (do_switch) {
    	raw_spin_lock(&ctx->lock);
-		perf_ctx_disable(ctx, false);
+		perf_ctx_disable(ctx, 0);
inside_switch:
    	perf_ctx_sched_task_cb(ctx, false);
    	task_ctx_sched_out(ctx, NULL, EVENT_ALL);
-		perf_ctx_enable(ctx, false);
+		perf_ctx_enable(ctx, 0);
    	raw_spin_unlock(&ctx->lock);
    }
 }
@@ -3981,11 +3990,9 @@ static void
 ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
 {
    struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+	enum event_type_t active_type = event_type & ~EVENT_FLAGS;
    struct perf_event_pmu_context *pmu_ctx;
    int is_active = ctx->is_active;
-	bool cgroup = event_type & EVENT_CGROUP;
-
-	event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock);
@@ -4003,7 +4010,7 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t
    	barrier();
    }
-	ctx->is_active |= (event_type | EVENT_TIME);
+	ctx->is_active |= active_type | EVENT_TIME;
    if (ctx->task) {
    	if (!(is_active & EVENT_ALL))
    		cpuctx->task_ctx = ctx;
@@ -4018,13 +4025,13 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t
     * in order to give them the best chance of going on.
     */
    if (is_active & EVENT_PINNED) {
-		for_each_epc(pmu_ctx, ctx, pmu, cgroup)
+		for_each_epc(pmu_ctx, ctx, pmu, event_type)
    		__pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED);
    }
/* Then walk through the lower prio flexible groups */
    if (is_active & EVENT_FLEXIBLE) {
-		for_each_epc(pmu_ctx, ctx, pmu, cgroup)
+		for_each_epc(pmu_ctx, ctx, pmu, event_type)
    		__pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE);
    }
 }
@@ -4041,11 +4048,11 @@ static void perf_event_context_sched_in(struct task_struct *task)
if (cpuctx->task_ctx == ctx) {
    	perf_ctx_lock(cpuctx, ctx);
-		perf_ctx_disable(ctx, false);
+		perf_ctx_disable(ctx, 0);
perf_ctx_sched_task_cb(ctx, true);
-		perf_ctx_enable(ctx, false);
+		perf_ctx_enable(ctx, 0);
    	perf_ctx_unlock(cpuctx, ctx);
    	goto rcu_unlock;
    }
@@ -4058,7 +4065,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
    if (!ctx->nr_events)
    	goto unlock;
-	perf_ctx_disable(ctx, false);
+	perf_ctx_disable(ctx, 0);
    /*
     * We want to keep the following priority order:
     * cpu pinned (that don't need to move), task pinned,
@@ -4068,7 +4075,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
     * events, no need to flip the cpuctx's events around.
     */
    if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
-		perf_ctx_disable(&cpuctx->ctx, false);
+		perf_ctx_disable(&cpuctx->ctx, 0);
    	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);
    }
@@ -4077,9 +4084,9 @@ static void perf_event_context_sched_in(struct task_struct *task)
    perf_ctx_sched_task_cb(cpuctx->task_ctx, true);
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
-		perf_ctx_enable(&cpuctx->ctx, false);
+		perf_ctx_enable(&cpuctx->ctx, 0);
-	perf_ctx_enable(ctx, false);
+	perf_ctx_enable(ctx, 0);
unlock:
    perf_ctx_unlock(cpuctx, ctx);
-- 
2.49.0.395.g12beb8f557-goog



    

2025

2024

2023

2022

2021

2020

2019

2018

2017

[PATCH v4 02/38] perf: Skip pmu_ctx based on event_type