Vendor extensions are maintained in riscv_isa_vendor (separate from standard extensions which live in riscv_isa). Create vendor variants for the existing extension helpers to interface with the riscv_isa_vendor bitmap. There is a good amount of overlap between these functions, so the alternative checking code can be factored out.
Signed-off-by: Charlie Jenkins charlie@rivosinc.com --- arch/riscv/errata/sifive/errata.c | 2 + arch/riscv/errata/thead/errata.c | 2 + arch/riscv/include/asm/cpufeature.h | 142 +++++++++++++++++++++++++++--------- arch/riscv/include/asm/hwprobe.h | 3 + arch/riscv/kernel/cpufeature.c | 53 ++++++++++++-- arch/riscv/kernel/sys_hwprobe.c | 4 +- 6 files changed, 161 insertions(+), 45 deletions(-)
diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index 3d9a32d791f7..847ff85cc911 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -99,6 +99,8 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, for (alt = begin; alt < end; alt++) { if (alt->vendor_id != SIFIVE_VENDOR_ID) continue; + if (alt->patch_id >= RISCV_ISA_VENDOR_EXT_BASE) + continue; if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) { WARN(1, "This errata id:%d is not in kernel errata list", alt->patch_id); continue; diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index b1c410bbc1ae..6e3eabfe92af 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -163,6 +163,8 @@ void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, for (alt = begin; alt < end; alt++) { if (alt->vendor_id != THEAD_VENDOR_ID) continue; + if (alt->patch_id >= RISCV_ISA_VENDOR_EXT_BASE) + continue; if (alt->patch_id >= ERRATA_THEAD_NUMBER) continue;
diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 50fa174cccb9..12dd36bafa2a 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -110,23 +110,19 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i #define riscv_isa_extension_available(isa_bitmap, ext) \ __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
+bool __riscv_isa_vendor_extension_available(const unsigned long *vendor_isa_bitmap, + unsigned int bit); +#define riscv_isa_vendor_extension_available(isa_bitmap, ext) \ + __riscv_isa_vendor_extension_available(isa_bitmap, RISCV_ISA_VENDOR_EXT_##ext) + static __always_inline bool -riscv_has_extension_likely(const unsigned long ext) +__riscv_has_extension_likely_alternatives(const unsigned long vendor, const unsigned long ext) { - compiletime_assert(ext < RISCV_ISA_EXT_MAX, - "ext must be < RISCV_ISA_EXT_MAX"); - - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { - asm goto( - ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1) - : - : [ext] "i" (ext) - : - : l_no); - } else { - if (!__riscv_isa_extension_available(NULL, ext)) - goto l_no; - } + asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1) + : + : [vendor] "i" (vendor), [ext] "i" (ext) + : + : l_no);
return true; l_no: @@ -134,42 +130,118 @@ riscv_has_extension_likely(const unsigned long ext) }
static __always_inline bool -riscv_has_extension_unlikely(const unsigned long ext) +__riscv_has_extension_unlikely_alternatives(const unsigned long vendor, const unsigned long ext) { - compiletime_assert(ext < RISCV_ISA_EXT_MAX, - "ext must be < RISCV_ISA_EXT_MAX"); - - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { - asm goto( - ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1) - : - : [ext] "i" (ext) - : - : l_yes); - } else { - if (__riscv_isa_extension_available(NULL, ext)) - goto l_yes; - } + asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1) + : + : [vendor] "i" (vendor), [ext] "i" (ext) + : + : l_yes);
return false; l_yes: return true; }
+/* Standard extension helpers */ + +static __always_inline bool +riscv_has_extension_likely(const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_likely_alternatives(0, ext); + else + return __riscv_isa_extension_available(NULL, ext); +} + +static __always_inline bool +riscv_has_extension_unlikely(const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_unlikely_alternatives(0, ext); + else + return __riscv_isa_extension_available(NULL, ext); +} + static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext) { - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && riscv_has_extension_likely(ext)) - return true; + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX");
- return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && + __riscv_has_extension_likely_alternatives(0, ext)) + return true; + else + return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); }
static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext) { - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && riscv_has_extension_unlikely(ext)) + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && + __riscv_has_extension_unlikely_alternatives(0, ext)) return true; + else + return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); +} + +/* Vendor extension helpers */ + +static __always_inline bool +riscv_has_vendor_extension_likely(const unsigned long vendor, const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX, + "ext must be < RISCV_ISA_VENDOR_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_likely_alternatives(vendor, ext); + else + return __riscv_isa_vendor_extension_available(NULL, ext); +}
- return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); +static __always_inline bool +riscv_has_vendor_extension_unlikely(const unsigned long vendor, const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX, + "ext must be < RISCV_ISA_VENDOR_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_unlikely_alternatives(vendor, ext); + else + return __riscv_isa_vendor_extension_available(NULL, ext); +} + +static __always_inline bool riscv_cpu_has_vendor_extension_likely(const unsigned long vendor, + int cpu, const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX, + "ext must be < RISCV_ISA_VENDOR_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_likely_alternatives(vendor, ext); + else + return __riscv_isa_vendor_extension_available(hart_isa_vendor[cpu].isa, ext); +} + +static __always_inline bool riscv_cpu_has_vendor_extension_unlikely(const unsigned long vendor, + int cpu, + const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_VENDOR_EXT_MAX, + "ext must be < RISCV_ISA_VENDOR_EXT_MAX"); + + if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) + return __riscv_has_extension_unlikely_alternatives(vendor, ext); + else + return __riscv_isa_vendor_extension_available(hart_isa_vendor[cpu].isa, ext); }
#endif diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 630507dff5ea..1378c3c9401a 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -6,6 +6,7 @@ #ifndef _ASM_HWPROBE_H #define _ASM_HWPROBE_H
+#include <linux/cpumask.h> #include <uapi/asm/hwprobe.h>
#define RISCV_HWPROBE_MAX_KEY 6 @@ -39,4 +40,6 @@ static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, return pair->value == other_pair->value; }
+void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus); + #endif diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index d7a33e017a15..799ec2d2e9e0 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -20,6 +20,7 @@ #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwcap.h> +#include <asm/hwprobe.h> #include <asm/patch.h> #include <asm/processor.h> #include <asm/sbi.h> @@ -82,6 +83,30 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i } EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
+/** + * __riscv_isa_vendor_extension_available() - Check whether given vendor + * extension is available or not. The vendor extension must be associated + * with the same vendor that was used to populate isa_bitmap. + * + * @isa_bitmap: ISA bitmap to use + * @bit: bit position of the desired extension + * Return: true or false + * + * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. + */ +bool __riscv_isa_vendor_extension_available(const unsigned long *isa_bitmap, unsigned int bit) +{ + const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa_vendor; + + bit -= RISCV_ISA_VENDOR_EXT_BASE; + + if (bit < 0 || bit >= RISCV_ISA_VENDOR_EXT_MAX) + return false; + + return test_bit(bit, bmap) ? true : false; +} +EXPORT_SYMBOL_GPL(__riscv_isa_vendor_extension_available); + static bool riscv_isa_extension_check(int id) { switch (id) { @@ -832,25 +857,37 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, { struct alt_entry *alt; void *oldptr, *altptr; - u16 id, value; + u16 id, value, vendor;
if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) return;
- for (alt = begin; alt < end; alt++) { - if (alt->vendor_id != 0) - continue; + struct riscv_hwprobe mvendorid = { + .key = RISCV_HWPROBE_KEY_MVENDORID, + .value = 0 + };
+ hwprobe_arch_id(&mvendorid, cpu_possible_mask); + + for (alt = begin; alt < end; alt++) { id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); + vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);
- if (id >= RISCV_ISA_EXT_MAX) { + if (id >= RISCV_ISA_VENDOR_EXT_BASE) { + if (vendor != mvendorid.value || + !__riscv_isa_vendor_extension_available(NULL, id)) + continue; + } else if (id < RISCV_ISA_EXT_MAX) { + if (alt->vendor_id != 0) + continue; + + if (!__riscv_isa_extension_available(NULL, id)) + continue; + } else { WARN(1, "This extension id:%d is not in ISA extension list", id); continue; }
- if (!__riscv_isa_extension_available(NULL, id)) - continue; - value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); if (!riscv_cpufeature_patch_check(id, value)) continue; diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 8cae41a502dd..394f1343490c 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -13,11 +13,11 @@ #include <asm/uaccess.h> #include <asm/unistd.h> #include <asm/vector.h> +#include <asm/vendor/thead.h> #include <vdso/vsyscall.h>
-static void hwprobe_arch_id(struct riscv_hwprobe *pair, - const struct cpumask *cpus) +void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus) { u64 id = -1ULL; bool first = true;