Choosing the initial state of static branches changes the assembly layout (if the condition is expected to be likely, inline, or unlikely, out of line via a jump). The _TRUE/_FALSE defines for CONFIG_INIT_ON_*_DEFAULT_ON were accidentally removed. These need to stay so that the CONFIG controls the pessimization of the resulting static branch NOP/JMP locations.
Fixes: 04013513cc84 ("mm, page_alloc: do not rely on the order of page_poison and init_on_alloc/free parameters") Cc: stable@vger.kernel.org Signed-off-by: Kees Cook keescook@chromium.org --- include/linux/mm.h | 8 ++++++++ mm/page_alloc.c | 8 ++++++++ 2 files changed, 16 insertions(+)
diff --git a/include/linux/mm.h b/include/linux/mm.h index 77e64e3eac80..b3317d91ee8e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2871,7 +2871,11 @@ static inline void kernel_poison_pages(struct page *page, int numpages) { } static inline void kernel_unpoison_pages(struct page *page, int numpages) { } #endif
+#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON +DECLARE_STATIC_KEY_TRUE(init_on_alloc); +#else DECLARE_STATIC_KEY_FALSE(init_on_alloc); +#endif static inline bool want_init_on_alloc(gfp_t flags) { if (static_branch_unlikely(&init_on_alloc)) @@ -2879,7 +2883,11 @@ static inline bool want_init_on_alloc(gfp_t flags) return flags & __GFP_ZERO; }
+#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON +DECLARE_STATIC_KEY_TRUE(init_on_free); +#else DECLARE_STATIC_KEY_FALSE(init_on_free); +#endif static inline bool want_init_on_free(void) { return static_branch_unlikely(&init_on_free); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3e4b29ee2b1e..f2d474a844cf 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -167,10 +167,18 @@ unsigned long totalcma_pages __read_mostly;
int percpu_pagelist_fraction; gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; +#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON +DEFINE_STATIC_KEY_TRUE(init_on_alloc); +#else DEFINE_STATIC_KEY_FALSE(init_on_alloc); +#endif EXPORT_SYMBOL(init_on_alloc);
+#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON +DEFINE_STATIC_KEY_TRUE(init_on_free); +#else DEFINE_STATIC_KEY_FALSE(init_on_free); +#endif EXPORT_SYMBOL(init_on_free);
static bool _init_on_alloc_enabled_early __read_mostly