From: Frederic Weisbecker frederic@kernel.org
[ Upstream commit e7f2be115f0746b969c0df14c0d182f65f005ca5 ]
getrusage(RUSAGE_THREAD) with nohz_full may return shorter utime/stime than the actual time.
task_cputime_adjusted() snapshots utime and stime and then adjust their sum to match the scheduler maintained cputime.sum_exec_runtime. Unfortunately in nohz_full, sum_exec_runtime is only updated once per second in the worst case, causing a discrepancy against utime and stime that can be updated anytime by the reader using vtime.
To fix this situation, perform an update of cputime.sum_exec_runtime when the cputime snapshot reports the task as actually running while the tick is disabled. The related overhead is then contained within the relevant situations.
Reported-by: Hasegawa Hitomi hasegawa-hitomi@fujitsu.com Signed-off-by: Frederic Weisbecker frederic@kernel.org Signed-off-by: Hasegawa Hitomi hasegawa-hitomi@fujitsu.com Signed-off-by: Thomas Gleixner tglx@linutronix.de Tested-by: Masayoshi Mizuma m.mizuma@jp.fujitsu.com Acked-by: Phil Auld pauld@redhat.com Link: https://lore.kernel.org/r/20211026141055.57358-3-frederic@kernel.org Signed-off-by: Sasha Levin sashal@kernel.org --- include/linux/sched/cputime.h | 5 +++-- kernel/sched/cputime.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index d0677f6739f63..787f3221b575b 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -18,15 +18,16 @@ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -extern void task_cputime(struct task_struct *t, +extern bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime); extern u64 task_gtime(struct task_struct *t); #else -static inline void task_cputime(struct task_struct *t, +static inline bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime) { *utime = t->utime; *stime = t->stime; + return false; }
static inline u64 task_gtime(struct task_struct *t) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 45c2cd37fe6b0..2d6c3372d8795 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -664,7 +664,8 @@ void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) .sum_exec_runtime = p->se.sum_exec_runtime, };
- task_cputime(p, &cputime.utime, &cputime.stime); + if (task_cputime(p, &cputime.utime, &cputime.stime)) + cputime.sum_exec_runtime = task_sched_runtime(p); cputime_adjust(&cputime, &p->prev_cputime, ut, st); } EXPORT_SYMBOL_GPL(task_cputime_adjusted); @@ -857,19 +858,21 @@ u64 task_gtime(struct task_struct *t) * add up the pending nohz execution time since the last * cputime snapshot. */ -void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) +bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime) { struct vtime *vtime = &t->vtime; unsigned int seq; u64 delta; + int ret;
if (!vtime_accounting_enabled()) { *utime = t->utime; *stime = t->stime; - return; + return false; }
do { + ret = false; seq = read_seqcount_begin(&vtime->seqcount);
*utime = t->utime; @@ -879,6 +882,7 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) if (vtime->state == VTIME_INACTIVE || is_idle_task(t)) continue;
+ ret = true; delta = vtime_delta(vtime);
/* @@ -890,5 +894,7 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) else if (vtime->state == VTIME_SYS) *stime += vtime->stime + delta; } while (read_seqcount_retry(&vtime->seqcount, seq)); + + return ret; } #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */