With /proc/pid/maps now being read under per-vma lock protection we can reuse parts of that code to execute PROCMAP_QUERY ioctl also without taking mmap_lock. The change is designed to reduce mmap_lock contention and prevent PROCMAP_QUERY ioctl calls from blocking address space updates.
This patchset was split out of the original patchset [1] that introduced per-vma lock usage for /proc/pid/maps reading. It contains PROCMAP_QUERY tests, code refactoring patch to simplify the main change and the actual transition to per-vma lock.
[1] https://lore.kernel.org/all/20250704060727.724817-1-surenb@google.com/
Suren Baghdasaryan (3): selftests/proc: test PROCMAP_QUERY ioctl while vma is concurrently modified fs/proc/task_mmu: factor out proc_maps_private fields used by PROCMAP_QUERY fs/proc/task_mmu: execute PROCMAP_QUERY ioctl under per-vma locks
fs/proc/internal.h | 15 +- fs/proc/task_mmu.c | 149 ++++++++++++------ tools/testing/selftests/proc/proc-maps-race.c | 65 ++++++++ 3 files changed, 174 insertions(+), 55 deletions(-)
base-commit: 01da54f10fddf3b01c5a3b80f6b16bbad390c302
Extend /proc/pid/maps tearing tests to verify PROCMAP_QUERY ioctl operation correctness while the vma is being concurrently modified.
Signed-off-by: Suren Baghdasaryan surenb@google.com --- tools/testing/selftests/proc/proc-maps-race.c | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+)
diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 66773685a047..d40854a07ec1 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -32,6 +32,8 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include <linux/fs.h> +#include <sys/ioctl.h> #include <sys/mman.h> #include <sys/stat.h> #include <sys/types.h> @@ -317,6 +319,25 @@ static bool capture_mod_pattern(FIXTURE_DATA(proc_maps_race) *self, strcmp(restored_first_line->text, self->first_line.text) == 0; }
+static bool query_addr_at(int maps_fd, void *addr, + unsigned long *vma_start, unsigned long *vma_end) +{ + struct procmap_query q; + + memset(&q, 0, sizeof(q)); + q.size = sizeof(q); + /* Find the VMA at the split address */ + q.query_addr = (unsigned long long)addr; + q.query_flags = 0; + if (ioctl(maps_fd, PROCMAP_QUERY, &q)) + return false; + + *vma_start = q.vma_start; + *vma_end = q.vma_end; + + return true; +} + static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self) { return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot | PROT_EXEC, @@ -559,6 +580,8 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) do { bool last_line_changed; bool first_line_changed; + unsigned long vma_start; + unsigned long vma_end;
ASSERT_TRUE(read_boundary_lines(self, &new_last_line, &new_first_line));
@@ -595,6 +618,19 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) first_line_changed = strcmp(new_first_line.text, self->first_line.text) != 0; ASSERT_EQ(last_line_changed, first_line_changed);
+ /* Check if PROCMAP_QUERY ioclt() finds the right VMA */ + ASSERT_TRUE(query_addr_at(self->maps_fd, mod_info->addr + self->page_size, + &vma_start, &vma_end)); + /* + * The vma at the split address can be either the same as + * original one (if read before the split) or the same as the + * first line in the second page (if read after the split). + */ + ASSERT_TRUE((vma_start == self->last_line.start_addr && + vma_end == self->last_line.end_addr) || + (vma_start == split_first_line.start_addr && + vma_end == split_first_line.end_addr)); + clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); } while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec); @@ -636,6 +672,9 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) clock_gettime(CLOCK_MONOTONIC_COARSE, &start_ts); start_test_loop(&start_ts, self->verbose); do { + unsigned long vma_start; + unsigned long vma_end; + ASSERT_TRUE(read_boundary_lines(self, &new_last_line, &new_first_line));
/* Check if we read vmas after shrinking it */ @@ -662,6 +701,16 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize) "Expand result invalid", self)); }
+ /* Check if PROCMAP_QUERY ioclt() finds the right VMA */ + ASSERT_TRUE(query_addr_at(self->maps_fd, mod_info->addr, &vma_start, &vma_end)); + /* + * The vma should stay at the same address and have either the + * original size of 3 pages or 1 page if read after shrinking. + */ + ASSERT_TRUE(vma_start == self->last_line.start_addr && + (vma_end - vma_start == self->page_size * 3 || + vma_end - vma_start == self->page_size)); + clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); } while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec); @@ -703,6 +752,9 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) clock_gettime(CLOCK_MONOTONIC_COARSE, &start_ts); start_test_loop(&start_ts, self->verbose); do { + unsigned long vma_start; + unsigned long vma_end; + ASSERT_TRUE(read_boundary_lines(self, &new_last_line, &new_first_line));
/* Check if we read vmas after remapping it */ @@ -729,6 +781,19 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap) "Remap restore result invalid", self)); }
+ /* Check if PROCMAP_QUERY ioclt() finds the right VMA */ + ASSERT_TRUE(query_addr_at(self->maps_fd, mod_info->addr + self->page_size, + &vma_start, &vma_end)); + /* + * The vma should either stay at the same address and have the + * original size of 3 pages or we should find the remapped vma + * at the remap destination address with size of 1 page. + */ + ASSERT_TRUE((vma_start == self->last_line.start_addr && + vma_end - vma_start == self->page_size * 3) || + (vma_start == self->last_line.start_addr + self->page_size && + vma_end - vma_start == self->page_size)); + clock_gettime(CLOCK_MONOTONIC_COARSE, &end_ts); end_test_iteration(&end_ts, self->verbose); } while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec);
On Thu, 31 Jul 2025 15:00:22 -0700 Suren Baghdasaryan surenb@google.com wrote:
Extend /proc/pid/maps tearing tests to verify PROCMAP_QUERY ioctl operation correctness while the vma is being concurrently modified.
Signed-off-by: Suren Baghdasaryan surenb@google.com
Acked-by: SeongJae Park sj@kernel.org Tested-by: SeongJae Park sj@kernel.org
Thanks, SJ
[...]
On Fri, Aug 1, 2025 at 6:38 PM SeongJae Park sj@kernel.org wrote:
On Thu, 31 Jul 2025 15:00:22 -0700 Suren Baghdasaryan surenb@google.com wrote:
Extend /proc/pid/maps tearing tests to verify PROCMAP_QUERY ioctl operation correctness while the vma is being concurrently modified.
Signed-off-by: Suren Baghdasaryan surenb@google.com
Acked-by: SeongJae Park sj@kernel.org Tested-by: SeongJae Park sj@kernel.org
Thanks SJ! I'll include this in my next respin.
Thanks, SJ
[...]
Refactor struct proc_maps_private so that the fields used by PROCMAP_QUERY ioctl are moved into a separate structure. In the next patch this allows ioctl to reuse some of the functions used for reading /proc/pid/maps without using file->private_data. This prevents concurrent modification of file->private_data members by ioctl and /proc/pid/maps readers.
The change is pure code refactoring and has no functional changes.
Signed-off-by: Suren Baghdasaryan surenb@google.com --- fs/proc/internal.h | 15 ++++++---- fs/proc/task_mmu.c | 70 +++++++++++++++++++++++----------------------- 2 files changed, 45 insertions(+), 40 deletions(-)
diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 7c235451c5ea..e2447b22592e 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -379,16 +379,21 @@ extern void proc_self_init(void); * task_[no]mmu.c */ struct mem_size_stats; -struct proc_maps_private { - struct inode *inode; - struct task_struct *task; + +struct proc_maps_query_data { struct mm_struct *mm; - struct vma_iterator iter; - loff_t last_pos; #ifdef CONFIG_PER_VMA_LOCK bool mmap_locked; struct vm_area_struct *locked_vma; #endif +}; + +struct proc_maps_private { + struct inode *inode; + struct task_struct *task; + struct vma_iterator iter; + loff_t last_pos; + struct proc_maps_query_data query; #ifdef CONFIG_NUMA struct mempolicy *task_mempolicy; #endif diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 3d6d8a9f13fc..509fa162760a 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -132,11 +132,11 @@ static void release_task_mempolicy(struct proc_maps_private *priv)
#ifdef CONFIG_PER_VMA_LOCK
-static void unlock_vma(struct proc_maps_private *priv) +static void unlock_vma(struct proc_maps_query_data *query) { - if (priv->locked_vma) { - vma_end_read(priv->locked_vma); - priv->locked_vma = NULL; + if (query->locked_vma) { + vma_end_read(query->locked_vma); + query->locked_vma = NULL; } }
@@ -151,14 +151,14 @@ static inline bool lock_vma_range(struct seq_file *m, * walking the vma tree under rcu read protection. */ if (m->op != &proc_pid_maps_op) { - if (mmap_read_lock_killable(priv->mm)) + if (mmap_read_lock_killable(priv->query.mm)) return false;
- priv->mmap_locked = true; + priv->query.mmap_locked = true; } else { rcu_read_lock(); - priv->locked_vma = NULL; - priv->mmap_locked = false; + priv->query.locked_vma = NULL; + priv->query.mmap_locked = false; }
return true; @@ -166,10 +166,10 @@ static inline bool lock_vma_range(struct seq_file *m,
static inline void unlock_vma_range(struct proc_maps_private *priv) { - if (priv->mmap_locked) { - mmap_read_unlock(priv->mm); + if (priv->query.mmap_locked) { + mmap_read_unlock(priv->query.mm); } else { - unlock_vma(priv); + unlock_vma(&priv->query); rcu_read_unlock(); } } @@ -179,13 +179,13 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, { struct vm_area_struct *vma;
- if (priv->mmap_locked) + if (priv->query.mmap_locked) return vma_next(&priv->iter);
- unlock_vma(priv); - vma = lock_next_vma(priv->mm, &priv->iter, last_pos); + unlock_vma(&priv->query); + vma = lock_next_vma(priv->query.mm, &priv->iter, last_pos); if (!IS_ERR_OR_NULL(vma)) - priv->locked_vma = vma; + priv->query.locked_vma = vma;
return vma; } @@ -193,14 +193,14 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, loff_t pos) { - if (priv->mmap_locked) + if (priv->query.mmap_locked) return false;
rcu_read_unlock(); - mmap_read_lock(priv->mm); + mmap_read_lock(priv->query.mm); /* Reinitialize the iterator after taking mmap_lock */ vma_iter_set(&priv->iter, pos); - priv->mmap_locked = true; + priv->query.mmap_locked = true;
return true; } @@ -210,12 +210,12 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, static inline bool lock_vma_range(struct seq_file *m, struct proc_maps_private *priv) { - return mmap_read_lock_killable(priv->mm) == 0; + return mmap_read_lock_killable(priv->query.mm) == 0; }
static inline void unlock_vma_range(struct proc_maps_private *priv) { - mmap_read_unlock(priv->mm); + mmap_read_unlock(priv->query.mm); }
static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, @@ -258,7 +258,7 @@ static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) *ppos = vma->vm_end; } else { *ppos = SENTINEL_VMA_GATE; - vma = get_gate_vma(priv->mm); + vma = get_gate_vma(priv->query.mm); }
return vma; @@ -278,7 +278,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) if (!priv->task) return ERR_PTR(-ESRCH);
- mm = priv->mm; + mm = priv->query.mm; if (!mm || !mmget_not_zero(mm)) { put_task_struct(priv->task); priv->task = NULL; @@ -318,7 +318,7 @@ static void *m_next(struct seq_file *m, void *v, loff_t *ppos) static void m_stop(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; - struct mm_struct *mm = priv->mm; + struct mm_struct *mm = priv->query.mm;
if (!priv->task) return; @@ -339,9 +339,9 @@ static int proc_maps_open(struct inode *inode, struct file *file, return -ENOMEM;
priv->inode = inode; - priv->mm = proc_mem_open(inode, PTRACE_MODE_READ); - if (IS_ERR_OR_NULL(priv->mm)) { - int err = priv->mm ? PTR_ERR(priv->mm) : -ESRCH; + priv->query.mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR_OR_NULL(priv->query.mm)) { + int err = priv->query.mm ? PTR_ERR(priv->query.mm) : -ESRCH;
seq_release_private(inode, file); return err; @@ -355,8 +355,8 @@ static int proc_map_release(struct inode *inode, struct file *file) struct seq_file *seq = file->private_data; struct proc_maps_private *priv = seq->private;
- if (priv->mm) - mmdrop(priv->mm); + if (priv->query.mm) + mmdrop(priv->query.mm);
return seq_release_private(inode, file); } @@ -610,7 +610,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) if (!!karg.build_id_size != !!karg.build_id_addr) return -EINVAL;
- mm = priv->mm; + mm = priv->query.mm; if (!mm || !mmget_not_zero(mm)) return -ESRCH;
@@ -1307,7 +1307,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; struct mem_size_stats mss = {}; - struct mm_struct *mm = priv->mm; + struct mm_struct *mm = priv->query.mm; struct vm_area_struct *vma; unsigned long vma_start = 0, last_vma_end = 0; int ret = 0; @@ -1452,9 +1452,9 @@ static int smaps_rollup_open(struct inode *inode, struct file *file) goto out_free;
priv->inode = inode; - priv->mm = proc_mem_open(inode, PTRACE_MODE_READ); - if (IS_ERR_OR_NULL(priv->mm)) { - ret = priv->mm ? PTR_ERR(priv->mm) : -ESRCH; + priv->query.mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR_OR_NULL(priv->query.mm)) { + ret = priv->query.mm ? PTR_ERR(priv->query.mm) : -ESRCH;
single_release(inode, file); goto out_free; @@ -1472,8 +1472,8 @@ static int smaps_rollup_release(struct inode *inode, struct file *file) struct seq_file *seq = file->private_data; struct proc_maps_private *priv = seq->private;
- if (priv->mm) - mmdrop(priv->mm); + if (priv->query.mm) + mmdrop(priv->query.mm);
kfree(priv); return single_release(inode, file);
On 8/1/25 00:00, Suren Baghdasaryan wrote:
Refactor struct proc_maps_private so that the fields used by PROCMAP_QUERY ioctl are moved into a separate structure. In the next patch this allows ioctl to reuse some of the functions used for reading /proc/pid/maps without using file->private_data. This prevents concurrent modification of file->private_data members by ioctl and /proc/pid/maps readers.
The change is pure code refactoring and has no functional changes.
I think you'll need to adjust task_nommu.c as well, minimally I see it also has m_start() acceding priv->mm directly so it won't compile now?
Also not sure about the naming, struct is named "proc_maps_query_data" and priv field named "query" but the read() implementation uses it too, via priv->query, although it does no PROCMAP_QUERY.
Seems to me it's actually something like a mm+vma locking context? Which can be either stored in proc_maps_private for read() operations, or local on-stack for ioctl().
On Fri, Aug 1, 2025 at 3:55 AM Vlastimil Babka vbabka@suse.cz wrote:
On 8/1/25 00:00, Suren Baghdasaryan wrote:
Refactor struct proc_maps_private so that the fields used by PROCMAP_QUERY ioctl are moved into a separate structure. In the next patch this allows ioctl to reuse some of the functions used for reading /proc/pid/maps without using file->private_data. This prevents concurrent modification of file->private_data members by ioctl and /proc/pid/maps readers.
The change is pure code refactoring and has no functional changes.
I think you'll need to adjust task_nommu.c as well, minimally I see it also has m_start() acceding priv->mm directly so it won't compile now?
Ugh, yes, you are right. I'll need to adjust NOMMU code as well. And kernel test bot seems to be complaining already :)
Also not sure about the naming, struct is named "proc_maps_query_data" and priv field named "query" but the read() implementation uses it too, via priv->query, although it does no PROCMAP_QUERY.
Seems to me it's actually something like a mm+vma locking context? Which can be either stored in proc_maps_private for read() operations, or local on-stack for ioctl().
Yes, I struggled with the naming of this structure. Any help with this is highly appreciated.
Hi Suren,
kernel test robot noticed the following build errors:
[auto build test ERROR on 01da54f10fddf3b01c5a3b80f6b16bbad390c302]
url: https://github.com/intel-lab-lkp/linux/commits/Suren-Baghdasaryan/selftests-... base: 01da54f10fddf3b01c5a3b80f6b16bbad390c302 patch link: https://lore.kernel.org/r/20250731220024.702621-3-surenb%40google.com patch subject: [PATCH 2/3] fs/proc/task_mmu: factor out proc_maps_private fields used by PROCMAP_QUERY config: riscv-randconfig-002-20250801 (https://download.01.org/0day-ci/archive/20250801/202508012010.9IA7JflG-lkp@i...) compiler: clang version 17.0.6 (https://github.com/llvm/llvm-project 6009708b4367171ccdbf4b5905cb6a803753fe18) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250801/202508012010.9IA7JflG-lkp@i...)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot lkp@intel.com | Closes: https://lore.kernel.org/oe-kbuild-all/202508012010.9IA7JflG-lkp@intel.com/
All errors (new ones prefixed by >>):
fs/proc/task_nommu.c:207:13: error: no member named 'mm' in 'struct proc_maps_private'
207 | mm = priv->mm; | ~~~~ ^ fs/proc/task_nommu.c:229:31: error: no member named 'mm' in 'struct proc_maps_private' 229 | struct mm_struct *mm = priv->mm; | ~~~~ ^ fs/proc/task_nommu.c:262:8: error: no member named 'mm' in 'struct proc_maps_private' 262 | priv->mm = proc_mem_open(inode, PTRACE_MODE_READ); | ~~~~ ^ fs/proc/task_nommu.c:263:27: error: no member named 'mm' in 'struct proc_maps_private' 263 | if (IS_ERR_OR_NULL(priv->mm)) { | ~~~~ ^ fs/proc/task_nommu.c:264:19: error: no member named 'mm' in 'struct proc_maps_private' 264 | int err = priv->mm ? PTR_ERR(priv->mm) : -ESRCH; | ~~~~ ^ fs/proc/task_nommu.c:264:38: error: no member named 'mm' in 'struct proc_maps_private' 264 | int err = priv->mm ? PTR_ERR(priv->mm) : -ESRCH; | ~~~~ ^ fs/proc/task_nommu.c:279:12: error: no member named 'mm' in 'struct proc_maps_private' 279 | if (priv->mm) | ~~~~ ^ fs/proc/task_nommu.c:280:16: error: no member named 'mm' in 'struct proc_maps_private' 280 | mmdrop(priv->mm); | ~~~~ ^ 8 errors generated.
vim +207 fs/proc/task_nommu.c
fe441980161751 Ben Wolsieffer 2023-09-15 191 fe441980161751 Ben Wolsieffer 2023-09-15 192 static void *m_start(struct seq_file *m, loff_t *ppos) ^1da177e4c3f41 Linus Torvalds 2005-04-16 193 { dbf8685c8e2140 David Howells 2006-09-27 194 struct proc_maps_private *priv = m->private; fe441980161751 Ben Wolsieffer 2023-09-15 195 unsigned long last_addr = *ppos; dbf8685c8e2140 David Howells 2006-09-27 196 struct mm_struct *mm; 0c563f14804356 Matthew Wilcox (Oracle 2022-09-06 197) fe441980161751 Ben Wolsieffer 2023-09-15 198 /* See proc_get_vma(). Zero at the start or after lseek. */ fe441980161751 Ben Wolsieffer 2023-09-15 199 if (last_addr == -1UL) 0c563f14804356 Matthew Wilcox (Oracle 2022-09-06 200) return NULL; dbf8685c8e2140 David Howells 2006-09-27 201 dbf8685c8e2140 David Howells 2006-09-27 202 /* pin the task and mm whilst we play with them */ 2c03376d2db005 Oleg Nesterov 2014-10-09 203 priv->task = get_proc_task(priv->inode); dbf8685c8e2140 David Howells 2006-09-27 204 if (!priv->task) ec6fd8a4355cda Al Viro 2011-02-15 205 return ERR_PTR(-ESRCH); dbf8685c8e2140 David Howells 2006-09-27 206 27692cd56e2aa6 Oleg Nesterov 2014-10-09 @207 mm = priv->mm; 578d7699e5c2ad Ben Wolsieffer 2023-09-14 208 if (!mm || !mmget_not_zero(mm)) { 578d7699e5c2ad Ben Wolsieffer 2023-09-14 209 put_task_struct(priv->task); 578d7699e5c2ad Ben Wolsieffer 2023-09-14 210 priv->task = NULL; 27692cd56e2aa6 Oleg Nesterov 2014-10-09 211 return NULL; 578d7699e5c2ad Ben Wolsieffer 2023-09-14 212 } dbf8685c8e2140 David Howells 2006-09-27 213 d8ed45c5dcd455 Michel Lespinasse 2020-06-08 214 if (mmap_read_lock_killable(mm)) { 8a713e7df3352b Konstantin Khlebnikov 2019-07-11 215 mmput(mm); 578d7699e5c2ad Ben Wolsieffer 2023-09-14 216 put_task_struct(priv->task); 578d7699e5c2ad Ben Wolsieffer 2023-09-14 217 priv->task = NULL; 8a713e7df3352b Konstantin Khlebnikov 2019-07-11 218 return ERR_PTR(-EINTR); 8a713e7df3352b Konstantin Khlebnikov 2019-07-11 219 } 8a713e7df3352b Konstantin Khlebnikov 2019-07-11 220 fe441980161751 Ben Wolsieffer 2023-09-15 221 vma_iter_init(&priv->iter, mm, last_addr); 47fecca15c0944 Oleg Nesterov 2014-10-09 222 fe441980161751 Ben Wolsieffer 2023-09-15 223 return proc_get_vma(priv, ppos); dbf8685c8e2140 David Howells 2006-09-27 224 } dbf8685c8e2140 David Howells 2006-09-27 225
Utilize per-vma locks to stabilize vma after lookup without taking mmap_lock during PROCMAP_QUERY ioctl execution. If vma lock is contended, we fall back to mmap_lock but take it only momentarily to lock the vma and release the mmap_lock. In a very unlikely case of vm_refcnt overflow, this fall back path will fail and ioctl is done under mmap_lock protection.
This change is designed to reduce mmap_lock contention and prevent PROCMAP_QUERY ioctl calls from blocking address space updates.
Signed-off-by: Suren Baghdasaryan surenb@google.com --- fs/proc/task_mmu.c | 81 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 16 deletions(-)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 509fa162760a..b504b798e8fe 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -517,28 +517,78 @@ static int pid_maps_open(struct inode *inode, struct file *file) PROCMAP_QUERY_VMA_FLAGS \ )
-static int query_vma_setup(struct mm_struct *mm) +#ifdef CONFIG_PER_VMA_LOCK + +static int query_vma_setup(struct proc_maps_query_data *query) { - return mmap_read_lock_killable(mm); + query->locked_vma = NULL; + query->mmap_locked = false; + + return 0; }
-static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma) +static void query_vma_teardown(struct proc_maps_query_data *query) { - mmap_read_unlock(mm); + if (query->mmap_locked) + mmap_read_unlock(query->mm); + else + unlock_vma(query); }
-static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr) +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_query_data *query, + unsigned long addr) { - return find_vma(mm, addr); + struct vm_area_struct *vma; + struct vma_iterator vmi; + + unlock_vma(query); + rcu_read_lock(); + vma_iter_init(&vmi, query->mm, addr); + vma = lock_next_vma(query->mm, &vmi, addr); + rcu_read_unlock(); + + if (!IS_ERR_OR_NULL(vma)) { + query->locked_vma = vma; + } else if (PTR_ERR(vma) == -EAGAIN) { + /* Fallback to mmap_lock on vma->vm_refcnt overflow */ + mmap_read_lock(query->mm); + vma = find_vma(query->mm, addr); + query->mmap_locked = true; + } + + return vma; }
-static struct vm_area_struct *query_matching_vma(struct mm_struct *mm, +#else /* CONFIG_PER_VMA_LOCK */ + +static int query_vma_setup(struct proc_maps_query_data *query) +{ + return mmap_read_lock_killable(query->mm); +} + +static void query_vma_teardown(struct proc_maps_query_data *query) +{ + mmap_read_unlock(query->mm); +} + +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_query_data *query, + unsigned long addr) +{ + return find_vma(query->mm, addr); +} + +#endif /* CONFIG_PER_VMA_LOCK */ + +static struct vm_area_struct *query_matching_vma(struct proc_maps_query_data *query, unsigned long addr, u32 flags) { struct vm_area_struct *vma;
next_vma: - vma = query_vma_find_by_addr(mm, addr); + vma = query_vma_find_by_addr(query, addr); + if (IS_ERR(vma)) + return vma; + if (!vma) goto no_vma;
@@ -579,11 +629,11 @@ static struct vm_area_struct *query_matching_vma(struct mm_struct *mm, return ERR_PTR(-ENOENT); }
-static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) +static int do_procmap_query(struct mm_struct *mm, void __user *uarg) { + struct proc_maps_query_data query = { .mm = mm }; struct procmap_query karg; struct vm_area_struct *vma; - struct mm_struct *mm; const char *name = NULL; char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; __u64 usize; @@ -610,17 +660,16 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) if (!!karg.build_id_size != !!karg.build_id_addr) return -EINVAL;
- mm = priv->query.mm; if (!mm || !mmget_not_zero(mm)) return -ESRCH;
- err = query_vma_setup(mm); + err = query_vma_setup(&query); if (err) { mmput(mm); return err; }
- vma = query_matching_vma(mm, karg.query_addr, karg.query_flags); + vma = query_matching_vma(&query, karg.query_addr, karg.query_flags); if (IS_ERR(vma)) { err = PTR_ERR(vma); vma = NULL; @@ -705,7 +754,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) }
/* unlock vma or mmap_lock, and put mm_struct before copying data to user */ - query_vma_teardown(mm, vma); + query_vma_teardown(&query); mmput(mm);
if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), @@ -725,7 +774,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg) return 0;
out: - query_vma_teardown(mm, vma); + query_vma_teardown(&query); mmput(mm); kfree(name_buf); return err; @@ -738,7 +787,7 @@ static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned l
switch (cmd) { case PROCMAP_QUERY: - return do_procmap_query(priv, (void __user *)arg); + return do_procmap_query(priv->query.mm, (void __user *)arg); default: return -ENOIOCTLCMD; }
linux-kselftest-mirror@lists.linaro.org
-
kernel test robot -
SeongJae Park -
Suren Baghdasaryan -
Vlastimil Babka