The memory pinning code in uaccess_with_memcpy.c does not check
for HugeTLB or THP pmds, and will enter an infinite loop should
a __copy_to_user or __clear_user occur against a huge page.
This patch adds detection code for huge pages to pin_page_for_write.
As this code can be executed in a fast path it refers to the actual
pmds rather than the vma. If a HugeTLB or THP is found (they have
the same pmd representation on ARM), the page table spinlock is
taken to prevent modification whilst the page is pinned.
On ARM, huge pages are only represented as pmds, thus no huge pud
checks are performed. (For huge puds one would lock the page table
in a similar manner as in the pmd case).
Two helper functions are introduced; pmd_thp_or_huge will check
whether or not a page is huge or transparent huge (which have the
same pmd layout on ARM), and pmd_hugewillfault will detect whether
or not a page fault will occur on write to the page.
Running the following test (with the chunking from read_zero
removed):
$ dd if=/dev/zero of=/dev/null bs=10M count=1024
Gave: 2.3 GB/s backed by normal pages,
2.9 GB/s backed by huge pages,
5.1 GB/s backed by huge pages, with page mask=HPAGE_MASK.
After some discussion, it was decided not to adopt the HPAGE_MASK,
as this would have a significant detrimental effect on the overall
system latency due to page_table_lock being held for too long.
This could be revisited if split huge page locks are adopted.
Signed-off-by: Steve Capper <steve.capper(a)linaro.org>
---
Changes in V3:
pmd_hugewillfault and pmd_thp_or_huge were erroneously ommitted from
pgtable-2level.h causing build problems. Empty stubs are added for now
as we don't have huge page support for short descriptors.
Nicolas, can I please re-add your Reviewed-by to this?
---
arch/arm/include/asm/pgtable-2level.h | 7 ++++++
arch/arm/include/asm/pgtable-3level.h | 3 +++
arch/arm/lib/uaccess_with_memcpy.c | 41 ++++++++++++++++++++++++++++++++---
3 files changed, 48 insertions(+), 3 deletions(-)
diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h
index f97ee02..86a659a 100644
--- a/arch/arm/include/asm/pgtable-2level.h
+++ b/arch/arm/include/asm/pgtable-2level.h
@@ -181,6 +181,13 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
+/*
+ * We don't have huge page support for short descriptors, for the moment
+ * define empty stubs for use by pin_page_for_write.
+ */
+#define pmd_hugewillfault(pmd) (0)
+#define pmd_thp_or_huge(pmd) (0)
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_PGTABLE_2LEVEL_H */
diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index 5689c18..39c54cf 100644
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -206,6 +206,9 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
#define __HAVE_ARCH_PMD_WRITE
#define pmd_write(pmd) (!(pmd_val(pmd) & PMD_SECT_RDONLY))
+#define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd))
+#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
#define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)
diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
index 025f742..3e58d71 100644
--- a/arch/arm/lib/uaccess_with_memcpy.c
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -18,6 +18,7 @@
#include <linux/hardirq.h> /* for in_atomic() */
#include <linux/gfp.h>
#include <linux/highmem.h>
+#include <linux/hugetlb.h>
#include <asm/current.h>
#include <asm/page.h>
@@ -40,7 +41,35 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
return 0;
pmd = pmd_offset(pud, addr);
- if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
+ if (unlikely(pmd_none(*pmd)))
+ return 0;
+
+ /*
+ * A pmd can be bad if it refers to a HugeTLB or THP page.
+ *
+ * Both THP and HugeTLB pages have the same pmd layout
+ * and should not be manipulated by the pte functions.
+ *
+ * Lock the page table for the destination and check
+ * to see that it's still huge and whether or not we will
+ * need to fault on write, or if we have a splitting THP.
+ */
+ if (unlikely(pmd_thp_or_huge(*pmd))) {
+ ptl = ¤t->mm->page_table_lock;
+ spin_lock(ptl);
+ if (unlikely(!pmd_thp_or_huge(*pmd)
+ || pmd_hugewillfault(*pmd)
+ || pmd_trans_splitting(*pmd))) {
+ spin_unlock(ptl);
+ return 0;
+ }
+
+ *ptep = NULL;
+ *ptlp = ptl;
+ return 1;
+ }
+
+ if (unlikely(pmd_bad(*pmd)))
return 0;
pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
@@ -94,7 +123,10 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n)
from += tocopy;
n -= tocopy;
- pte_unmap_unlock(pte, ptl);
+ if (pte)
+ pte_unmap_unlock(pte, ptl);
+ else
+ spin_unlock(ptl);
}
if (!atomic)
up_read(¤t->mm->mmap_sem);
@@ -147,7 +179,10 @@ __clear_user_memset(void __user *addr, unsigned long n)
addr += tocopy;
n -= tocopy;
- pte_unmap_unlock(pte, ptl);
+ if (pte)
+ pte_unmap_unlock(pte, ptl);
+ else
+ spin_unlock(ptl);
}
up_read(¤t->mm->mmap_sem);
--
1.8.1.4
Hi Mark, Alex
I have some big.LITTLE MP updates from ARM for LSK can you please pull
these for this month's LSK release? I believe these go into the LSK
topic branch v3.10/topic/big.LITTLE.
Thanks
--
Tixy
The following changes since commit 7ac3860c98608e2f9782902b4439e0ade53c2364:
Merge tag 'big-LITTLE-MP-13.08' into for-lsk (2013-09-05 18:17:48 +0100)
are available in the git repository at:
git://git.linaro.org/arm/big.LITTLE/mp.git for-lsk
for you to fetch changes up to 68f98fec62bada031c89ee61b5bccd61914c23c3:
Merge tag 'big-LITTLE-MP-13.10' into for-lsk (2013-10-11 17:12:02 +0100)
----------------------------------------------------------------
Chris Redpath (7):
sched: HMP: Change default HMP thresholds
sched: HMP: Additional trace points for debugging HMP behaviour
arm: ipi raise/start/end tracing
smp: smp_cross_call function pointer tracing
sched: HMP: fix potential logical errors
hmp: Remove potential for task_struct access race
HMP: Implement task packing for small tasks in HMP systems
Jon Medhurst (1):
Merge tag 'big-LITTLE-MP-13.10' into for-lsk
arch/arm/Kconfig | 12 ++
arch/arm/kernel/smp.c | 5 +
drivers/irqchip/irq-gic.c | 5 +-
include/trace/events/arm-ipi.h | 100 +++++++++++++
include/trace/events/sched.h | 72 ++++++++++
include/trace/events/smp.h | 91 ++++++++++++
kernel/sched/fair.c | 302 +++++++++++++++++++++++++++++-----------
kernel/smp.c | 12 +-
8 files changed, 514 insertions(+), 85 deletions(-)
create mode 100644 include/trace/events/arm-ipi.h
create mode 100644 include/trace/events/smp.h
From: Vijaya Kumar K <Vijaya.Kumar(a)caviumnetworks.com>
Based on the step-handler and break-handler hooks patch from
Sandeepa (Patch 1), KGDB debugging support is added for EL1
debug in AArch64 mode. Any updates that come for Patch 1 from
Sandeepa will be rebased in next version
With first patch,register layout is updated to be inline with GDB tool.
Basic GDB connection, break point set/clear and info commands
are supported except step/next debugging
With second patch, step/next debugging support is added, where in
pc is updated to point to the instruction to be stepped and
stopped.
v2:
- Moved break instruction encoding to debug-monitors.h file
- Fixed endianess of compile break instruction encoding
- Updated I/O buffer sizes
- Updated register buffer size
- Remove changes to debug_exception handler in entry.S for
- ELR update and step debugging with update pc instead of ELR
- Rebased against AArch64 upstream kernel
v1:
- Initial patch-set
Tested with Aarch64 GDB tool chain on simulator
Sandeepa Prabhu (1):
AArch64: Add single-step and breakpoint handler hooks
Vijaya Kumar K (2):
AArch64: KGDB: Add Basic KGDB support
AArch64: KGDB: Add step debugging support
arch/arm64/include/asm/debug-monitors.h | 30 +++
arch/arm64/include/asm/kgdb.h | 81 ++++++++
arch/arm64/kernel/Makefile | 1 +
arch/arm64/kernel/debug-monitors.c | 95 ++++++++-
arch/arm64/kernel/entry.S | 6 +-
arch/arm64/kernel/kgdb.c | 341 +++++++++++++++++++++++++++++++
6 files changed, 550 insertions(+), 4 deletions(-)
create mode 100644 arch/arm64/include/asm/kgdb.h
create mode 100644 arch/arm64/kernel/kgdb.c
--
1.7.9.5
