November 2022 - Linux-stable-mirror

[PATCH 4.19 1/3] bitops: always define asm-generic non-atomic bitops

by Mikulas Patocka

commit 21bb8af513d35c005c401706030f4eb469538d1d upstream. Move generic non-atomic bitops from the asm-generic header which gets included only when there are no architecture-specific alternatives, to a separate independent file to make them always available. Almost no actual code changes, only one comment added to generic_test_bit() saying that it's an atomic operation itself and thus `volatile` must always stay there with no cast-aways. Suggested-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> # comment Suggested-by: Marco Elver <elver(a)google.com> # reference to kernel-doc Signed-off-by: Alexander Lobakin <alexandr.lobakin(a)intel.com> Reviewed-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Reviewed-by: Marco Elver <elver(a)google.com> Signed-off-by: Yury Norov <yury.norov(a)gmail.com> --- include/asm-generic/bitops/generic-non-atomic.h | 130 ++++++++++++++++++++++++ 1 file changed, 130 insertions(+) Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:16:08.000000000 +0100 @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H + +#include <linux/bits.h> + +#ifndef _LINUX_BITOPS_H +#error only <linux/bitops.h> can be included directly +#endif + +/* + * Generic definitions for bit operations, should not be used in regular code + * directly. + */ + +/** + * generic___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static __always_inline void +generic___clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * generic___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline +void generic___change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * generic___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * generic___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static __always_inline int +generic___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * generic_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline int +generic_test_bit(unsigned int nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH 5.4 3/3] provide arch_test_bit_acquire for architectures that define test_bit

by Mikulas Patocka

commit d6ffe6067a54972564552ea45d320fb98db1ac5e upstream. Some architectures define their own arch_test_bit and they also need arch_test_bit_acquire, otherwise they won't compile. We also clean up the code by using the generic test_bit if that is equivalent to the arch-specific version. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Cc: stable(a)vger.kernel.org Fixes: 8238b4579866 ("wait_on_bit: add an acquire memory barrier") Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/alpha/include/asm/bitops.h | 8 +++----- arch/arc/include/asm/bitops.h | 17 +++-------------- arch/h8300/include/asm/bitops.h | 3 ++- arch/hexagon/include/asm/bitops.h | 15 +++++++++++++++ arch/ia64/include/asm/bitops.h | 8 +++----- arch/m68k/include/asm/bitops.h | 8 +++----- arch/s390/include/asm/bitops.h | 12 +++--------- arch/sh/include/asm/bitops-op32.h | 11 ++--------- 8 files changed, 34 insertions(+), 48 deletions(-) Index: linux-stable/arch/alpha/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/alpha/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/alpha/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -8,6 +8,7 @@ #include <asm/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Copyright 1994, Linus Torvalds. @@ -283,11 +284,8 @@ __test_and_change_bit(unsigned long nr, return (old & mask) != 0; } -static inline int -test_bit(int nr, const volatile void * addr) -{ - return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /* * ffz = Find First Zero in word. Undefined if no zero exists, Index: linux-stable/arch/hexagon/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/hexagon/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/hexagon/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -172,7 +172,22 @@ static inline int __test_bit(int nr, con return retval; } +static inline int __test_bit_acquire(int nr, const volatile unsigned long *addr) +{ + int retval; + + asm volatile( + "{P0 = tstbit(%1,%2); if (P0.new) %0 = #1; if (!P0.new) %0 = #0;}\n" + : "=&r" (retval) + : "r" (addr[BIT_WORD(nr)]), "r" (nr % BITS_PER_LONG) + : "p0", "memory" + ); + + return retval; +} + #define test_bit(nr, addr) __test_bit(nr, addr) +#define test_bit_acquire(nr, addr) __test_bit_acquire(nr, addr) /* * ffz - find first zero in word. Index: linux-stable/arch/ia64/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/ia64/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/ia64/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -18,6 +18,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /** * set_bit - Atomically set a bit in memory @@ -331,11 +332,8 @@ __test_and_change_bit (int nr, void *add return (old & bit) != 0; } -static __inline__ int -test_bit (int nr, const volatile void *addr) -{ - return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /** * ffz - find the first zero bit in a long word Index: linux-stable/arch/m68k/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/m68k/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/m68k/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -14,6 +14,7 @@ #include <linux/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Bit access functions vary across the ColdFire and 68k families. @@ -148,11 +149,8 @@ static inline void bfchg_mem_change_bit( #define __change_bit(nr, vaddr) change_bit(nr, vaddr) -static inline int test_bit(int nr, const volatile unsigned long *vaddr) -{ - return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0; -} - +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) Index: linux-stable/arch/s390/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/s390/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/s390/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -38,6 +38,7 @@ #include <linux/types.h> #include <asm/atomic_ops.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG) @@ -209,15 +210,8 @@ static inline bool arch___test_and_chang return (ch >> (nr & 7)) & 1; } -static inline bool arch_test_bit(unsigned long nr, - const volatile unsigned long *ptr) -{ - const volatile unsigned char *addr; - - addr = ((const volatile unsigned char *)ptr); - addr += (nr ^ (BITS_PER_LONG - 8)) >> 3; - return (*addr >> (nr & 7)) & 1; -} +#define arch_test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire static inline bool arch_test_and_set_bit_lock(unsigned long nr, volatile unsigned long *ptr) Index: linux-stable/arch/sh/include/asm/bitops-op32.h =================================================================== --- linux-stable.orig/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:16:37.000000000 +0100 @@ -130,14 +130,7 @@ static inline int __test_and_change_bit( return (old & mask) != 0; } -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline int test_bit(int nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire #endif /* __ASM_SH_BITOPS_OP32_H */ Index: linux-stable/arch/h8300/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/h8300/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/h8300/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -83,7 +83,8 @@ static inline int test_bit(int nr, const return ret; } -#define __test_bit(nr, addr) test_bit(nr, addr) +#define __test_bit(nr, addr) test_bit(nr, addr) +#define test_bit_acquire(nr, addr) test_bit(nr, addr) #define H8300_GEN_TEST_BITOP(FNNAME, OP) \ static inline int FNNAME(int nr, void *addr) \ Index: linux-stable/arch/arc/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/arc/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 +++ linux-stable/arch/arc/include/asm/bitops.h 2022-11-14 22:16:37.000000000 +0100 @@ -18,6 +18,7 @@ #ifndef CONFIG_ARC_HAS_LLSC #include <asm/smp.h> #endif +#include <asm-generic/bitops/generic-non-atomic.h> #ifdef CONFIG_ARC_HAS_LLSC @@ -236,20 +237,8 @@ BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3) #endif -/* - * This routine doesn't need to be atomic. - */ -static inline int -test_bit(unsigned int nr, const volatile unsigned long *addr) -{ - unsigned long mask; - - addr += nr >> 5; - - mask = 1UL << (nr & 0x1f); - - return ((mask & *addr) != 0); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire #ifdef CONFIG_ISA_ARCOMPACT

2 years, 1 month

1
0
0 0

[PATCH 5.4 2/3] wait_on_bit: add an acquire memory barrier

by Mikulas Patocka

commit 8238b4579866b7c1bb99883cfe102a43db5506ff upstream. There are several places in the kernel where wait_on_bit is not followed by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read). On architectures with weak memory ordering, it may happen that memory accesses that follow wait_on_bit are reordered before wait_on_bit and they may return invalid data. Fix this class of bugs by introducing a new function "test_bit_acquire" that works like test_bit, but has acquire memory ordering semantics. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Acked-by: Will Deacon <will(a)kernel.org> Cc: stable(a)vger.kernel.org Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/x86/include/asm/bitops.h | 21 +++++++++++++++++++++ include/asm-generic/bitops/generic-non-atomic.h | 13 +++++++++++++ include/asm-generic/bitops/non-atomic.h | 14 ++++++++++++++ include/linux/buffer_head.h | 2 +- include/linux/wait_bit.h | 8 ++++---- kernel/sched/wait_bit.c | 2 +- 6 files changed, 54 insertions(+), 6 deletions(-) Index: linux-stable/arch/x86/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/x86/include/asm/bitops.h 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/arch/x86/include/asm/bitops.h 2022-11-15 14:26:07.000000000 +0100 @@ -207,6 +207,20 @@ static __always_inline bool constant_tes (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } +static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr) +{ + bool oldbit; + + asm volatile("testb %2,%1" + CC_SET(nz) + : CC_OUT(nz) (oldbit) + : "m" (((unsigned char *)addr)[nr >> 3]), + "i" (1 << (nr & 7)) + :"memory"); + + return oldbit; +} + static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) { bool oldbit; @@ -224,6 +238,13 @@ static __always_inline bool variable_tes ? constant_test_bit((nr), (addr)) \ : variable_test_bit((nr), (addr))) +static __always_inline bool +test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) : + variable_test_bit(nr, addr); +} + /** * __ffs - find first set bit in word * @word: The word to search Index: linux-stable/include/asm-generic/bitops/non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/non-atomic.h 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/non-atomic.h 2022-11-15 14:14:36.000000000 +0100 @@ -3,6 +3,7 @@ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #include <asm/types.h> +#include <asm/barrier.h> /** * __set_bit - Set a bit in memory @@ -106,4 +107,17 @@ static inline int test_bit(int nr, const return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * arch_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} +#define test_bit_acquire arch_test_bit_acquire + #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ Index: linux-stable/include/linux/buffer_head.h =================================================================== --- linux-stable.orig/include/linux/buffer_head.h 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/include/linux/buffer_head.h 2022-11-15 14:14:36.000000000 +0100 @@ -166,7 +166,7 @@ static __always_inline int buffer_uptoda * make it consistent with folio_test_uptodate * pairs with smp_mb__before_atomic in set_buffer_uptodate */ - return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0; + return test_bit_acquire(BH_Uptodate, &bh->b_state); } #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) Index: linux-stable/include/linux/wait_bit.h =================================================================== --- linux-stable.orig/include/linux/wait_bit.h 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/include/linux/wait_bit.h 2022-11-15 14:14:36.000000000 +0100 @@ -71,7 +71,7 @@ static inline int wait_on_bit(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait, @@ -96,7 +96,7 @@ static inline int wait_on_bit_io(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait_io, @@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, unsigned long timeout) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit_timeout(word, bit, bit_wait_timeout, @@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, action, mode); } Index: linux-stable/kernel/sched/wait_bit.c =================================================================== --- linux-stable.orig/kernel/sched/wait_bit.c 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/kernel/sched/wait_bit.c 2022-11-15 14:14:36.000000000 +0100 @@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) ret = (*action)(&wbq_entry->key, mode); - } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); + } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); finish_wait(wq_head, &wbq_entry->wq_entry); Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/generic-non-atomic.h 2022-11-15 14:14:36.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-15 14:14:36.000000000 +0100 @@ -4,6 +4,7 @@ #define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H #include <linux/bits.h> +#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly @@ -127,4 +128,16 @@ generic_test_bit(unsigned int nr, const return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} + #endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH 5.4 1/3] bitops: always define asm-generic non-atomic bitops

by Mikulas Patocka

commit 21bb8af513d35c005c401706030f4eb469538d1d upstream. Move generic non-atomic bitops from the asm-generic header which gets included only when there are no architecture-specific alternatives, to a separate independent file to make them always available. Almost no actual code changes, only one comment added to generic_test_bit() saying that it's an atomic operation itself and thus `volatile` must always stay there with no cast-aways. Suggested-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> # comment Suggested-by: Marco Elver <elver(a)google.com> # reference to kernel-doc Signed-off-by: Alexander Lobakin <alexandr.lobakin(a)intel.com> Reviewed-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Reviewed-by: Marco Elver <elver(a)google.com> Signed-off-by: Yury Norov <yury.norov(a)gmail.com> --- include/asm-generic/bitops/generic-non-atomic.h | 130 ++++++++++++++++++++++++ 1 file changed, 130 insertions(+) Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:16:35.000000000 +0100 @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H + +#include <linux/bits.h> + +#ifndef _LINUX_BITOPS_H +#error only <linux/bitops.h> can be included directly +#endif + +/* + * Generic definitions for bit operations, should not be used in regular code + * directly. + */ + +/** + * generic___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static __always_inline void +generic___clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * generic___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline +void generic___change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * generic___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * generic___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static __always_inline int +generic___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * generic_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline int +generic_test_bit(unsigned int nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH 5.10 3/3] provide arch_test_bit_acquire for architectures that define test_bit

by Mikulas Patocka

commit d6ffe6067a54972564552ea45d320fb98db1ac5e upstream. Some architectures define their own arch_test_bit and they also need arch_test_bit_acquire, otherwise they won't compile. We also clean up the code by using the generic test_bit if that is equivalent to the arch-specific version. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Cc: stable(a)vger.kernel.org Fixes: 8238b4579866 ("wait_on_bit: add an acquire memory barrier") Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/alpha/include/asm/bitops.h | 8 +++----- arch/arc/include/asm/bitops.h | 17 +++-------------- arch/h8300/include/asm/bitops.h | 3 ++- arch/hexagon/include/asm/bitops.h | 15 +++++++++++++++ arch/ia64/include/asm/bitops.h | 8 +++----- arch/m68k/include/asm/bitops.h | 8 +++----- arch/s390/include/asm/bitops.h | 12 +++--------- arch/sh/include/asm/bitops-op32.h | 11 ++--------- 8 files changed, 34 insertions(+), 48 deletions(-) Index: linux-stable/arch/alpha/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/alpha/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/alpha/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -8,6 +8,7 @@ #include <asm/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Copyright 1994, Linus Torvalds. @@ -283,11 +284,8 @@ __test_and_change_bit(unsigned long nr, return (old & mask) != 0; } -static inline int -test_bit(int nr, const volatile void * addr) -{ - return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /* * ffz = Find First Zero in word. Undefined if no zero exists, Index: linux-stable/arch/hexagon/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/hexagon/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/hexagon/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -172,7 +172,22 @@ static inline int __test_bit(int nr, con return retval; } +static inline int __test_bit_acquire(int nr, const volatile unsigned long *addr) +{ + int retval; + + asm volatile( + "{P0 = tstbit(%1,%2); if (P0.new) %0 = #1; if (!P0.new) %0 = #0;}\n" + : "=&r" (retval) + : "r" (addr[BIT_WORD(nr)]), "r" (nr % BITS_PER_LONG) + : "p0", "memory" + ); + + return retval; +} + #define test_bit(nr, addr) __test_bit(nr, addr) +#define test_bit_acquire(nr, addr) __test_bit_acquire(nr, addr) /* * ffz - find first zero in word. Index: linux-stable/arch/ia64/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/ia64/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/ia64/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -18,6 +18,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /** * set_bit - Atomically set a bit in memory @@ -331,11 +332,8 @@ __test_and_change_bit (int nr, void *add return (old & bit) != 0; } -static __inline__ int -test_bit (int nr, const volatile void *addr) -{ - return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /** * ffz - find the first zero bit in a long word Index: linux-stable/arch/m68k/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/m68k/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/m68k/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -14,6 +14,7 @@ #include <linux/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Bit access functions vary across the ColdFire and 68k families. @@ -148,11 +149,8 @@ static inline void bfchg_mem_change_bit( #define __change_bit(nr, vaddr) change_bit(nr, vaddr) -static inline int test_bit(int nr, const volatile unsigned long *vaddr) -{ - return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0; -} - +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) Index: linux-stable/arch/s390/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/s390/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/s390/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -38,6 +38,7 @@ #include <linux/types.h> #include <asm/atomic_ops.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG) @@ -209,15 +210,8 @@ static inline bool arch___test_and_chang return (ch >> (nr & 7)) & 1; } -static inline bool arch_test_bit(unsigned long nr, - const volatile unsigned long *ptr) -{ - const volatile unsigned char *addr; - - addr = ((const volatile unsigned char *)ptr); - addr += (nr ^ (BITS_PER_LONG - 8)) >> 3; - return (*addr >> (nr & 7)) & 1; -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire static inline bool arch_test_and_set_bit_lock(unsigned long nr, volatile unsigned long *ptr) Index: linux-stable/arch/sh/include/asm/bitops-op32.h =================================================================== --- linux-stable.orig/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:17:08.000000000 +0100 @@ -128,14 +128,7 @@ static inline int __test_and_change_bit( return (old & mask) != 0; } -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline int test_bit(int nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire #endif /* __ASM_SH_BITOPS_OP32_H */ Index: linux-stable/arch/h8300/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/h8300/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/h8300/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -83,7 +83,8 @@ static inline int test_bit(int nr, const return ret; } -#define __test_bit(nr, addr) test_bit(nr, addr) +#define __test_bit(nr, addr) test_bit(nr, addr) +#define test_bit_acquire(nr, addr) test_bit(nr, addr) #define H8300_GEN_TEST_BITOP(FNNAME, OP) \ static inline int FNNAME(int nr, void *addr) \ Index: linux-stable/arch/arc/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/arc/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 +++ linux-stable/arch/arc/include/asm/bitops.h 2022-11-14 22:17:08.000000000 +0100 @@ -18,6 +18,7 @@ #ifndef CONFIG_ARC_HAS_LLSC #include <asm/smp.h> #endif +#include <asm-generic/bitops/generic-non-atomic.h> #ifdef CONFIG_ARC_HAS_LLSC @@ -182,20 +183,8 @@ BIT_OPS(set, |, bset) BIT_OPS(clear, & ~, bclr) BIT_OPS(change, ^, bxor) -/* - * This routine doesn't need to be atomic. - */ -static inline int -test_bit(unsigned int nr, const volatile unsigned long *addr) -{ - unsigned long mask; - - addr += nr >> 5; - - mask = 1UL << (nr & 0x1f); - - return ((mask & *addr) != 0); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire #ifdef CONFIG_ISA_ARCOMPACT

2 years, 1 month

1
0
0 0

[PATCH 5.10 2/3] wait_on_bit: add an acquire memory barrier

by Mikulas Patocka

commit 8238b4579866b7c1bb99883cfe102a43db5506ff upstream. There are several places in the kernel where wait_on_bit is not followed by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read). On architectures with weak memory ordering, it may happen that memory accesses that follow wait_on_bit are reordered before wait_on_bit and they may return invalid data. Fix this class of bugs by introducing a new function "test_bit_acquire" that works like test_bit, but has acquire memory ordering semantics. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Acked-by: Will Deacon <will(a)kernel.org> Cc: stable(a)vger.kernel.org Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/x86/include/asm/bitops.h | 21 +++++++++++++++++++ include/asm-generic/bitops/generic-non-atomic.h | 13 +++++++++++ include/asm-generic/bitops/instrumented-non-atomic.h | 12 ++++++++++ include/asm-generic/bitops/non-atomic.h | 14 ++++++++++++ include/linux/buffer_head.h | 2 - include/linux/wait_bit.h | 8 +++---- kernel/sched/wait_bit.c | 2 - 7 files changed, 66 insertions(+), 6 deletions(-) Index: linux-stable/arch/x86/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/x86/include/asm/bitops.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/arch/x86/include/asm/bitops.h 2022-11-14 22:17:07.000000000 +0100 @@ -207,6 +207,20 @@ static __always_inline bool constant_tes (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } +static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr) +{ + bool oldbit; + + asm volatile("testb %2,%1" + CC_SET(nz) + : CC_OUT(nz) (oldbit) + : "m" (((unsigned char *)addr)[nr >> 3]), + "i" (1 << (nr & 7)) + :"memory"); + + return oldbit; +} + static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) { bool oldbit; @@ -224,6 +238,13 @@ static __always_inline bool variable_tes ? constant_test_bit((nr), (addr)) \ : variable_test_bit((nr), (addr))) +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) : + variable_test_bit(nr, addr); +} + /** * __ffs - find first set bit in word * @word: The word to search Index: linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/instrumented-non-atomic.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h 2022-11-14 22:17:07.000000000 +0100 @@ -135,4 +135,16 @@ static inline bool test_bit(long nr, con return arch_test_bit(nr, addr); } +/** + * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit_acquire(nr, addr); +} + #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ Index: linux-stable/include/asm-generic/bitops/non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/non-atomic.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/non-atomic.h 2022-11-14 22:17:07.000000000 +0100 @@ -3,6 +3,7 @@ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #include <asm/types.h> +#include <asm/barrier.h> /** * __set_bit - Set a bit in memory @@ -106,4 +107,17 @@ static inline int test_bit(int nr, const return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * arch_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} +#define test_bit_acquire arch_test_bit_acquire + #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ Index: linux-stable/include/linux/buffer_head.h =================================================================== --- linux-stable.orig/include/linux/buffer_head.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/include/linux/buffer_head.h 2022-11-14 22:17:07.000000000 +0100 @@ -166,7 +166,7 @@ static __always_inline int buffer_uptoda * make it consistent with folio_test_uptodate * pairs with smp_mb__before_atomic in set_buffer_uptodate */ - return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0; + return test_bit_acquire(BH_Uptodate, &bh->b_state); } #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) Index: linux-stable/include/linux/wait_bit.h =================================================================== --- linux-stable.orig/include/linux/wait_bit.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/include/linux/wait_bit.h 2022-11-14 22:17:07.000000000 +0100 @@ -71,7 +71,7 @@ static inline int wait_on_bit(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait, @@ -96,7 +96,7 @@ static inline int wait_on_bit_io(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait_io, @@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, unsigned long timeout) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit_timeout(word, bit, bit_wait_timeout, @@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, action, mode); } Index: linux-stable/kernel/sched/wait_bit.c =================================================================== --- linux-stable.orig/kernel/sched/wait_bit.c 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/kernel/sched/wait_bit.c 2022-11-14 22:17:07.000000000 +0100 @@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) ret = (*action)(&wbq_entry->key, mode); - } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); + } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); finish_wait(wq_head, &wbq_entry->wq_entry); Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:17:07.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:17:07.000000000 +0100 @@ -4,6 +4,7 @@ #define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H #include <linux/bits.h> +#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly @@ -127,4 +128,16 @@ generic_test_bit(unsigned int nr, const return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} + #endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH 5.10 1/3] bitops: always define asm-generic non-atomic bitops

by Mikulas Patocka

commit 21bb8af513d35c005c401706030f4eb469538d1d upstream. Move generic non-atomic bitops from the asm-generic header which gets included only when there are no architecture-specific alternatives, to a separate independent file to make them always available. Almost no actual code changes, only one comment added to generic_test_bit() saying that it's an atomic operation itself and thus `volatile` must always stay there with no cast-aways. Suggested-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> # comment Suggested-by: Marco Elver <elver(a)google.com> # reference to kernel-doc Signed-off-by: Alexander Lobakin <alexandr.lobakin(a)intel.com> Reviewed-by: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Reviewed-by: Marco Elver <elver(a)google.com> Signed-off-by: Yury Norov <yury.norov(a)gmail.com> --- include/asm-generic/bitops/generic-non-atomic.h | 130 ++++++++++++++++++++++++ 1 file changed, 130 insertions(+) Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:17:06.000000000 +0100 @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H + +#include <linux/bits.h> + +#ifndef _LINUX_BITOPS_H +#error only <linux/bitops.h> can be included directly +#endif + +/* + * Generic definitions for bit operations, should not be used in regular code + * directly. + */ + +/** + * generic___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static __always_inline void +generic___clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * generic___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline +void generic___change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * generic___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * generic___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline int +generic___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static __always_inline int +generic___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * generic_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline int +generic_test_bit(unsigned int nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH 5.15 3/3] provide arch_test_bit_acquire for architectures that define test_bit

by Mikulas Patocka

commit d6ffe6067a54972564552ea45d320fb98db1ac5e upstream. Some architectures define their own arch_test_bit and they also need arch_test_bit_acquire, otherwise they won't compile. We also clean up the code by using the generic test_bit if that is equivalent to the arch-specific version. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Cc: stable(a)vger.kernel.org Fixes: 8238b4579866 ("wait_on_bit: add an acquire memory barrier") Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/alpha/include/asm/bitops.h | 8 +++----- arch/h8300/include/asm/bitops.h | 3 ++- arch/hexagon/include/asm/bitops.h | 15 +++++++++++++++ arch/ia64/include/asm/bitops.h | 8 +++----- arch/m68k/include/asm/bitops.h | 8 +++----- arch/s390/include/asm/bitops.h | 11 +++-------- arch/sh/include/asm/bitops-op32.h | 11 ++--------- 7 files changed, 31 insertions(+), 33 deletions(-) Index: linux-stable/arch/alpha/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/alpha/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/alpha/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -8,6 +8,7 @@ #include <asm/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Copyright 1994, Linus Torvalds. @@ -283,11 +284,8 @@ __test_and_change_bit(unsigned long nr, return (old & mask) != 0; } -static inline int -test_bit(int nr, const volatile void * addr) -{ - return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /* * ffz = Find First Zero in word. Undefined if no zero exists, Index: linux-stable/arch/hexagon/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/hexagon/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/hexagon/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -172,7 +172,22 @@ static inline int __test_bit(int nr, con return retval; } +static inline int __test_bit_acquire(int nr, const volatile unsigned long *addr) +{ + int retval; + + asm volatile( + "{P0 = tstbit(%1,%2); if (P0.new) %0 = #1; if (!P0.new) %0 = #0;}\n" + : "=&r" (retval) + : "r" (addr[BIT_WORD(nr)]), "r" (nr % BITS_PER_LONG) + : "p0", "memory" + ); + + return retval; +} + #define test_bit(nr, addr) __test_bit(nr, addr) +#define test_bit_acquire(nr, addr) __test_bit_acquire(nr, addr) /* * ffz - find first zero in word. Index: linux-stable/arch/ia64/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/ia64/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/ia64/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -18,6 +18,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /** * set_bit - Atomically set a bit in memory @@ -331,11 +332,8 @@ __test_and_change_bit (int nr, void *add return (old & bit) != 0; } -static __inline__ int -test_bit (int nr, const volatile void *addr) -{ - return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire /** * ffz - find the first zero bit in a long word Index: linux-stable/arch/m68k/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/m68k/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/m68k/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -14,6 +14,7 @@ #include <linux/compiler.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> /* * Bit access functions vary across the ColdFire and 68k families. @@ -148,11 +149,8 @@ static inline void bfchg_mem_change_bit( #define __change_bit(nr, vaddr) change_bit(nr, vaddr) -static inline int test_bit(int nr, const volatile unsigned long *vaddr) -{ - return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0; -} - +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) Index: linux-stable/arch/s390/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/s390/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/s390/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -38,6 +38,7 @@ #include <linux/types.h> #include <asm/atomic_ops.h> #include <asm/barrier.h> +#include <asm-generic/bitops/generic-non-atomic.h> #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG) @@ -175,14 +176,8 @@ static inline bool arch___test_and_chang return old & mask; } -static inline bool arch_test_bit(unsigned long nr, - const volatile unsigned long *ptr) -{ - const volatile unsigned long *addr = __bitops_word(nr, ptr); - unsigned long mask = __bitops_mask(nr); - - return *addr & mask; -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire static inline bool arch_test_and_set_bit_lock(unsigned long nr, volatile unsigned long *ptr) Index: linux-stable/arch/sh/include/asm/bitops-op32.h =================================================================== --- linux-stable.orig/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/sh/include/asm/bitops-op32.h 2022-11-14 22:17:32.000000000 +0100 @@ -128,14 +128,7 @@ static inline int __test_and_change_bit( return (old & mask) != 0; } -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline int test_bit(int nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} +#define test_bit generic_test_bit +#define test_bit_acquire generic_test_bit_acquire #endif /* __ASM_SH_BITOPS_OP32_H */ Index: linux-stable/arch/h8300/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/h8300/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/h8300/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -87,7 +87,8 @@ static inline int test_bit(int nr, const return ret; } -#define __test_bit(nr, addr) test_bit(nr, addr) +#define __test_bit(nr, addr) test_bit(nr, addr) +#define test_bit_acquire(nr, addr) test_bit(nr, addr) #define H8300_GEN_TEST_BITOP(FNNAME, OP) \ static inline int FNNAME(int nr, void *addr) \

2 years, 1 month

1
0
0 0

[PATCH 5.15 2/3] wait_on_bit: add an acquire memory barrier

by Mikulas Patocka

commit 8238b4579866b7c1bb99883cfe102a43db5506ff upstream. There are several places in the kernel where wait_on_bit is not followed by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read). On architectures with weak memory ordering, it may happen that memory accesses that follow wait_on_bit are reordered before wait_on_bit and they may return invalid data. Fix this class of bugs by introducing a new function "test_bit_acquire" that works like test_bit, but has acquire memory ordering semantics. Signed-off-by: Mikulas Patocka <mpatocka(a)redhat.com> Acked-by: Will Deacon <will(a)kernel.org> Cc: stable(a)vger.kernel.org Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> --- arch/x86/include/asm/bitops.h | 21 +++++++++++++++++++ include/asm-generic/bitops/generic-non-atomic.h | 13 +++++++++++ include/asm-generic/bitops/instrumented-non-atomic.h | 12 ++++++++++ include/asm-generic/bitops/non-atomic.h | 3 ++ include/linux/buffer_head.h | 2 - include/linux/wait_bit.h | 8 +++---- kernel/sched/wait_bit.c | 2 - 7 files changed, 55 insertions(+), 6 deletions(-) Index: linux-stable/arch/x86/include/asm/bitops.h =================================================================== --- linux-stable.orig/arch/x86/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/arch/x86/include/asm/bitops.h 2022-11-14 22:17:32.000000000 +0100 @@ -207,6 +207,20 @@ static __always_inline bool constant_tes (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } +static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr) +{ + bool oldbit; + + asm volatile("testb %2,%1" + CC_SET(nz) + : CC_OUT(nz) (oldbit) + : "m" (((unsigned char *)addr)[nr >> 3]), + "i" (1 << (nr & 7)) + :"memory"); + + return oldbit; +} + static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) { bool oldbit; @@ -224,6 +238,13 @@ static __always_inline bool variable_tes ? constant_test_bit((nr), (addr)) \ : variable_test_bit((nr), (addr))) +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) : + variable_test_bit(nr, addr); +} + /** * __ffs - find first set bit in word * @word: The word to search Index: linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/instrumented-non-atomic.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h 2022-11-14 22:17:32.000000000 +0100 @@ -135,4 +135,16 @@ static inline bool test_bit(long nr, con return arch_test_bit(nr, addr); } +/** + * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit_acquire(nr, addr); +} + #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ Index: linux-stable/include/asm-generic/bitops/non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/non-atomic.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/non-atomic.h 2022-11-14 22:17:32.000000000 +0100 @@ -25,4 +25,7 @@ #define arch_test_bit generic_test_bit #define test_bit arch_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire +#define test_bit_acquire arch_test_bit_acquire + #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ Index: linux-stable/include/linux/buffer_head.h =================================================================== --- linux-stable.orig/include/linux/buffer_head.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/include/linux/buffer_head.h 2022-11-14 22:17:32.000000000 +0100 @@ -166,7 +166,7 @@ static __always_inline int buffer_uptoda * make it consistent with folio_test_uptodate * pairs with smp_mb__before_atomic in set_buffer_uptodate */ - return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0; + return test_bit_acquire(BH_Uptodate, &bh->b_state); } #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) Index: linux-stable/include/linux/wait_bit.h =================================================================== --- linux-stable.orig/include/linux/wait_bit.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/include/linux/wait_bit.h 2022-11-14 22:17:32.000000000 +0100 @@ -71,7 +71,7 @@ static inline int wait_on_bit(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait, @@ -96,7 +96,7 @@ static inline int wait_on_bit_io(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait_io, @@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, unsigned long timeout) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit_timeout(word, bit, bit_wait_timeout, @@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, action, mode); } Index: linux-stable/kernel/sched/wait_bit.c =================================================================== --- linux-stable.orig/kernel/sched/wait_bit.c 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/kernel/sched/wait_bit.c 2022-11-14 22:17:32.000000000 +0100 @@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) ret = (*action)(&wbq_entry->key, mode); - } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); + } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); finish_wait(wq_head, &wbq_entry->wq_entry); Index: linux-stable/include/asm-generic/bitops/generic-non-atomic.h =================================================================== --- linux-stable.orig/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:17:32.000000000 +0100 +++ linux-stable/include/asm-generic/bitops/generic-non-atomic.h 2022-11-14 22:17:32.000000000 +0100 @@ -4,6 +4,7 @@ #define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H #include <linux/bits.h> +#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly @@ -127,4 +128,16 @@ generic_test_bit(unsigned int nr, const return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} + #endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

2 years, 1 month

1
0
0 0

[PATCH v2] drm/i915/ttm: never purge busy objects

by Matthew Auld

In i915_gem_madvise_ioctl() we immediately purge the object is not currently used, like when the mm.pages are NULL. With shmem the pages might still be hanging around or are perhaps swapped out. Similarly with ttm we might still have the pages hanging around on the ttm resource, like with lmem or shmem, but here we need to be extra careful since async unbinds are possible as well as in-progress kernel moves. In i915_ttm_purge() we expect the pipeline-gutting to nuke the ttm resource for us, however if it's busy the memory is only moved to a ghost object, which then leads to broken behaviour when for example clearing the i915_tt->filp, since the actual ttm_tt is still alive and populated, even though it's been moved to the ghost object. When we later destroy the ghost object we hit the following, since the filp is now NULL: [ +0.006982] #PF: supervisor read access in kernel mode [ +0.005149] #PF: error_code(0x0000) - not-present page [ +0.005147] PGD 11631d067 P4D 11631d067 PUD 115972067 PMD 0 [ +0.005676] Oops: 0000 [#1] PREEMPT SMP NOPTI [ +0.012962] Workqueue: events ttm_device_delayed_workqueue [ttm] [ +0.006022] RIP: 0010:i915_ttm_tt_unpopulate+0x3a/0x70 [i915] [ +0.005879] Code: 89 fb 48 85 f6 74 11 8b 55 4c 48 8b 7d 30 45 31 c0 31 c9 e8 18 6a e5 e0 80 7d 60 00 74 20 48 8b 45 68 8b 55 08 4c 89 e7 5b 5d <48> 8b 40 20 83 e2 01 41 5c 89 d1 48 8b 70 30 e9 42 b2 ff ff 4c 89 [ +0.018782] RSP: 0000:ffffc9000bf6fd70 EFLAGS: 00010202 [ +0.005244] RAX: 0000000000000000 RBX: ffff8883e12ae380 RCX: 0000000000000000 [ +0.007150] RDX: 000000008000000e RSI: ffffffff823559b4 RDI: ffff8883e12ae3c0 [ +0.007142] RBP: ffff888103b65d48 R08: 0000000000000001 R09: 0000000000000001 [ +0.007144] R10: 0000000000000001 R11: ffff88829c2c8040 R12: ffff8883e12ae3c0 [ +0.007148] R13: 0000000000000001 R14: ffff888115184140 R15: ffff888115184248 [ +0.007154] FS: 0000000000000000(0000) GS:ffff88844db00000(0000) knlGS:0000000000000000 [ +0.008108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ +0.005763] CR2: 0000000000000020 CR3: 000000013fdb4004 CR4: 00000000003706e0 [ +0.007152] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ +0.007145] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ +0.007154] Call Trace: [ +0.002459] <TASK> [ +0.002126] ttm_tt_unpopulate.part.0+0x17/0x70 [ttm] [ +0.005068] ttm_bo_tt_destroy+0x1c/0x50 [ttm] [ +0.004464] ttm_bo_cleanup_memtype_use+0x25/0x40 [ttm] [ +0.005244] ttm_bo_cleanup_refs+0x90/0x2c0 [ttm] [ +0.004721] ttm_bo_delayed_delete+0x235/0x250 [ttm] [ +0.004981] ttm_device_delayed_workqueue+0x13/0x40 [ttm] [ +0.005422] process_one_work+0x248/0x560 [ +0.004028] worker_thread+0x4b/0x390 [ +0.003682] ? process_one_work+0x560/0x560 [ +0.004199] kthread+0xeb/0x120 [ +0.003163] ? kthread_complete_and_exit+0x20/0x20 [ +0.004815] ret_from_fork+0x1f/0x30 v2: - Just use ttm_bo_wait() directly (Niranjana) - Add testcase reference Testcase: igt@gem_madvise@dontneed-evict-race Fixes: 213d50927763 ("drm/i915/ttm: Introduce a TTM i915 gem object backend") Reported-by: Niranjana Vishwanathapura <niranjana.vishwanathapura(a)intel.com> Signed-off-by: Matthew Auld <matthew.auld(a)intel.com> Cc: Andrzej Hajda <andrzej.hajda(a)intel.com> Cc: Nirmoy Das <nirmoy.das(a)intel.com> Cc: <stable(a)vger.kernel.org> # v5.15+ Reviewed-by: Niranjana Vishwanathapura <niranjana.vishwanathapura(a)intel.com> --- drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index e4e55e3f4e41..34793863ea45 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -603,6 +603,10 @@ static int i915_ttm_truncate(struct drm_i915_gem_object *obj) WARN_ON_ONCE(obj->mm.madv == I915_MADV_WILLNEED); + err = ttm_bo_wait(bo, true, false); + if (err) + return err; + err = i915_ttm_move_notify(bo); if (err) return err; -- 2.38.1

2 years, 1 month

3
2
0 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

Linux-stable-mirror November 2022