Update the definiton of SMIDR_EL1 in the sysreg definition to reflect the information in DD0601 2025-06. This includes somewhat more generic ways of describing the sharing of SMCUs, more information on supported priorities and provides additional resolution for describing affinity groups.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/tools/sysreg | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 696ab1f32a67..9a1adb234ff0 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3659,11 +3659,15 @@ Field 3:0 BS EndSysreg

Sysreg SMIDR_EL1 3 1 0 0 6 -Res0 63:32 +Res0 63:60 +Field 59:56 NSMC +Field 55:52 HIP +Field 51:32 AFFINITY2 Field 31:24 IMPLEMENTER Field 23:16 REVISION Field 15 SMPS -Res0 14:12 +Field 14:13 SH +Res0 12 Field 11:0 AFFINITY EndSysreg

Some parts of the SME state are optional, enabled by additional features on top of the base FEAT_SME and controlled with enable bits in SMCR_ELx. We unconditionally enable these for the host but for KVM we will allow the feature set exposed to guests to be restricted by the VMM. These are the FFR register (FEAT_SME_FA64) and ZT0 (FEAT_SME2).

We defer saving of guest floating point state for non-protected guests to the host kernel. We also want to avoid having to reconfigure the guest floating point state if nothing used the floating point state while running the host. If the guest was running with the optional features disabled then traps will be enabled for them so the host kernel will need to skip accessing that state when saving state for the guest.

Support this by moving the decision about saving this state to the point where we bind floating point state to the CPU, adding a new variable to the cpu_fp_state which uses the enable bits in SMCR_ELx to flag which features are enabled.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/fpsimd.h | 1 + arch/arm64/kernel/fpsimd.c | 10 ++++++++-- arch/arm64/kvm/fpsimd.c | 1 + 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index b4359f942621..0ecdd7dcf623 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -87,6 +87,7 @@ struct cpu_fp_state { void *sme_state; u64 *svcr; u64 *fpmr; + u64 sme_features; unsigned int sve_vl; unsigned int sme_vl; enum fp_type *fp_type; diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 653c0dec6b18..77f9dfaffe8b 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -477,12 +477,12 @@ static void fpsimd_save_user_state(void)

if (*svcr & SVCR_ZA_MASK) sme_save_state(last->sme_state, - system_supports_sme2()); + last->sme_features & SMCR_ELx_EZT0);

/* If we are in streaming mode override regular SVE. */ if (*svcr & SVCR_SM_MASK) { save_sve_regs = true; - save_ffr = system_supports_fa64(); + save_ffr = last->sme_features & SMCR_ELx_FA64; vl = last->sme_vl; } } @@ -1655,6 +1655,12 @@ static void fpsimd_bind_task_to_cpu(void) last->to_save = FP_STATE_CURRENT; current->thread.fpsimd_cpu = smp_processor_id();

+ last->sme_features = 0; + if (system_supports_fa64()) + last->sme_features |= SMCR_ELx_FA64; + if (system_supports_sme2()) + last->sme_features |= SMCR_ELx_EZT0; + /* * Toggle SVE and SME trapping for userspace if needed, these * are serialsied by ret_to_user(). diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 15e17aca1dec..9158353d8be3 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -80,6 +80,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) fp_state.svcr = __ctxt_sys_reg(&vcpu->arch.ctxt, SVCR); fp_state.fpmr = __ctxt_sys_reg(&vcpu->arch.ctxt, FPMR); fp_state.fp_type = &vcpu->arch.fp_type; + fp_state.sme_features = 0;

if (vcpu_has_sve(vcpu)) fp_state.to_save = FP_STATE_SVE;

As with SVE we can only virtualise SME vector lengths that are supported by all CPUs in the system, implement similar checks to those for SVE. Since unlike SVE there are no specific vector lengths that are architecturally required the handling is subtly different, we report a system where this happens with a maximum vector length of -1.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/kernel/fpsimd.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index de2897d6208c..fbc586813f6a 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1244,7 +1244,8 @@ void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p) void __init sme_setup(void) { struct vl_info *info = &vl_info[ARM64_VEC_SME]; - int min_bit, max_bit; + DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); + int min_bit, max_bit, b;

if (!system_supports_sme()) return; @@ -1274,12 +1275,32 @@ void __init sme_setup(void) */ set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32));

+ bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map, + SVE_VQ_MAX); + + b = find_last_bit(tmp_map, SVE_VQ_MAX); + if (b >= SVE_VQ_MAX) + /* All VLs virtualisable */ + info->max_virtualisable_vl = SVE_VQ_MAX; + else if (b == SVE_VQ_MAX - 1) + /* No virtualisable VLs */ + info->max_virtualisable_vl = -1; + else + info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); + + if (info->max_virtualisable_vl > info->max_vl) + info->max_virtualisable_vl = info->max_vl; + pr_info("SME: minimum available vector length %u bytes per vector\n", info->min_vl); pr_info("SME: maximum available vector length %u bytes per vector\n", info->max_vl); pr_info("SME: default vector length %u bytes per vector\n", get_sme_default_vl()); + + /* KVM decides whether to support mismatched systems. Just warn here: */ + if (info->max_virtualisable_vl < info->max_vl) + pr_warn("SME: unvirtualisable vector lengths present\n"); }

void sme_suspend_exit(void)

The hypervisor copies of the SVE save and load functions are prototyped with third arguments specifying FFR should be accessed but the assembly functions overwrite whatever is supplied to unconditionally access FFR. Remove this and use the supplied parameter.

This has no effect currently since FFR is always present for SVE but will be important for SME.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/kvm/hyp/fpsimd.S | 2 -- 1 file changed, 2 deletions(-)

diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index e950875e31ce..6e16cbfc5df2 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -21,13 +21,11 @@ SYM_FUNC_START(__fpsimd_restore_state) SYM_FUNC_END(__fpsimd_restore_state)

SYM_FUNC_START(__sve_restore_state) - mov x2, #1 sve_load 0, x1, x2, 3 ret SYM_FUNC_END(__sve_restore_state)

SYM_FUNC_START(__sve_save_state) - mov x2, #1 sve_save 0, x1, x2, 3 ret SYM_FUNC_END(__sve_save_state)

In preparation for SME support move the macros used to access SVE state after the feature test macros, we will need to test for SME subfeatures to determine the size of the SME state.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 50 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index b501c2880ba2..8f48c2a488f7 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1045,31 +1045,6 @@ struct kvm_vcpu_arch { #define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))

-/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ -#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl)) - -#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) - -#define vcpu_sve_zcr_elx(vcpu) \ - (unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1) - -#define sve_state_size_from_vl(sve_max_vl) ({ \ - size_t __size_ret; \ - unsigned int __vq; \ - \ - if (WARN_ON(!sve_vl_valid(sve_max_vl))) { \ - __size_ret = 0; \ - } else { \ - __vq = sve_vq_from_vl(sve_max_vl); \ - __size_ret = SVE_SIG_REGS_SIZE(__vq); \ - } \ - \ - __size_ret; \ -}) - -#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.sve_max_vl) - #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ KVM_GUESTDBG_USE_SW_BP | \ KVM_GUESTDBG_USE_HW | \ @@ -1105,6 +1080,31 @@ struct kvm_vcpu_arch {

#define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs)

+/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ +#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ + sve_ffr_offset((vcpu)->arch.sve_max_vl)) + +#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) + +#define vcpu_sve_zcr_elx(vcpu) \ + (unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1) + +#define sve_state_size_from_vl(sve_max_vl) ({ \ + size_t __size_ret; \ + unsigned int __vq; \ + \ + if (WARN_ON(!sve_vl_valid(sve_max_vl))) { \ + __size_ret = 0; \ + } else { \ + __vq = sve_vq_from_vl(sve_max_vl); \ + __size_ret = SVE_SIG_REGS_SIZE(__vq); \ + } \ + \ + __size_ret; \ +}) + +#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.sve_max_vl) + /* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the * memory backed version of a register, and not the one most recently

SME, the Scalable Matrix Extension, is an arm64 extension which adds support for matrix operations, with core concepts patterned after SVE.

SVE introduced some complication in the ABI since it adds new vector floating point registers with runtime configurable size, the size being controlled by a prameter called the vector length (VL). To provide control of this to VMMs we offer two phase configuration of SVE, SVE must first be enabled for the vCPU with KVM_ARM_VCPU_INIT(KVM_ARM_VCPU_SVE), after which vector length may then be configured but the configurably sized floating point registers are inaccessible until finalized with a call to KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE) after which the configurably sized registers can be accessed.

SME introduces an additional independent configurable vector length which as well as controlling the size of the new ZA register also provides an alternative view of the configurably sized SVE registers (known as streaming mode) with the guest able to switch between the two modes as it pleases. There is also a fixed sized register ZT0 introduced in SME2. As well as streaming mode the guest may enable and disable ZA and (where SME2 is available) ZT0 dynamically independently of streaming mode. These modes are controlled via the system register SVCR.

We handle the configuration of the vector length for SME in a similar manner to SVE, requiring initialization and finalization of the feature with a pseudo register controlling the available SME vector lengths as for SVE. Further, if the guest has both SVE and SME then finalizing one prevents further configuration of the vector length for the other.

Where both SVE and SME are configured for the guest we always present the SVE registers to userspace as having the larger of the configured maximum SVE and SME vector lengths, discarding extra data at load time and zero padding on read as required if the active vector length is lower. Note that this means that enabling or disabling streaming mode while the guest is stopped will not zero Zn or Pn as it will when the guest is running, but it does allow SVCR, Zn and Pn to be read and written in any order.

Userspace access to ZA and (if configured) ZT0 is always available, they will be zeroed when the guest runs if disabled in SVCR and the value read will be zero if the guest stops with them disabled. This mirrors the behaviour of the architecture, enabling access causes ZA and ZT0 to be zeroed, while allowing access to SVCR, ZA and ZT0 to be performed in any order.

If SME is enabled for a guest without SVE then the FPSIMD Vn registers must be accessed via the low 128 bits of the SVE Zn registers as is the case when SVE is enabled. This is not ideal but allows access to SVCR and the registers in any order without duplication or ambiguity about which values should take effect. This may be an issue for VMMs that are unaware of SME on systems that implement it without SVE if they let SME be enabled, the lack of access to Vn may surprise them, but it seems like an unusual implementation choice.

For SME unware VMMs on systems with both SVE and SME support the SVE registers may be larger than expected, this should be less disruptive than on a system without SVE as they will simply ignore the high bits of the registers.

Signed-off-by: Mark Brown broonie@kernel.org --- Documentation/virt/kvm/api.rst | 117 +++++++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 6aa40ee05a4a..71f46b342641 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -406,7 +406,7 @@ Errors: instructions from device memory (arm64) ENOSYS data abort outside memslots with no syndrome info and KVM_CAP_ARM_NISV_TO_USER not enabled (arm64) - EPERM SVE feature set but not finalized (arm64) + EPERM SVE or SME feature set but not finalized (arm64) ======= ==============================================================

This ioctl is used to run a guest virtual cpu. While there are no @@ -2600,12 +2600,12 @@ Specifically: 0x6020 0000 0010 00d5 FPCR 32 fp_regs.fpcr ======================= ========= ===== =======================================

-.. [1] These encodings are not accepted for SVE-enabled vcpus. See - :ref:`KVM_ARM_VCPU_INIT`. +.. [1] These encodings are not accepted for SVE enabled vcpus. See + :ref:`KVM_ARM_VCPU_INIT`. They are also not accepted when SME is + enabled without SVE and the vcpu is in streaming mode.

The equivalent register content can be accessed via bits [127:0] of - the corresponding SVE Zn registers instead for vcpus that have SVE - enabled (see below). + the corresponding SVE Zn registers in these cases (see below).

arm64 CCSIDR registers are demultiplexed by CSSELR value::

@@ -2636,24 +2636,34 @@ arm64 SVE registers have the following bit patterns:: 0x6050 0000 0015 060 slice:5 FFR bits[256*slice + 255 : 256*slice] 0x6060 0000 0015 ffff KVM_REG_ARM64_SVE_VLS pseudo-register

-Access to register IDs where 2048 * slice >= 128 * max_vq will fail with -ENOENT. max_vq is the vcpu's maximum supported vector length in 128-bit -quadwords: see [2]_ below. +arm64 SME registers have the following bit patterns: + + 0x6080 0000 0017 00 <n:5> slice:5 ZA.H[n] bits[2048*slice + 2047 : 2048*slice] + 0x60XX 0000 0017 0100 ZT0 + 0x6060 0000 0017 fffe KVM_REG_ARM64_SME_VLS pseudo-register + +Access to Z, P or ZA register IDs where 2048 * slice >= 128 * max_vq +will fail with ENOENT. max_vq is the vcpu's maximum supported vector +length in 128-bit quadwords: see [2]_ below. + +Access to the ZA and ZT0 registers is only available if SVCR.ZA is set +to 1.

These registers are only accessible on vcpus for which SVE is enabled. See KVM_ARM_VCPU_INIT for details.

-In addition, except for KVM_REG_ARM64_SVE_VLS, these registers are not -accessible until the vcpu's SVE configuration has been finalized -using KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE). See KVM_ARM_VCPU_INIT -and KVM_ARM_VCPU_FINALIZE for more information about this procedure. +In addition, except for KVM_REG_ARM64_SVE_VLS and +KVM_REG_ARM64_SME_VLS, these registers are not accessible until the +vcpu's SVE and SME configuration has been finalized using +KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC). See KVM_ARM_VCPU_INIT and +KVM_ARM_VCPU_FINALIZE for more information about this procedure.

-KVM_REG_ARM64_SVE_VLS is a pseudo-register that allows the set of vector -lengths supported by the vcpu to be discovered and configured by -userspace. When transferred to or from user memory via KVM_GET_ONE_REG -or KVM_SET_ONE_REG, the value of this register is of type -__u64[KVM_ARM64_SVE_VLS_WORDS], and encodes the set of vector lengths as -follows:: +KVM_REG_ARM64_SVE_VLS and KVM_ARM64_VCPU_SME_VLS are pseudo-registers +that allows the set of vector lengths supported by the vcpu to be +discovered and configured by userspace. When transferred to or from +user memory via KVM_GET_ONE_REG or KVM_SET_ONE_REG, the value of this +register is of type __u64[KVM_ARM64_SVE_VLS_WORDS], and encodes the +set of vector lengths as follows::

__u64 vector_lengths[KVM_ARM64_SVE_VLS_WORDS];

@@ -2665,19 +2675,25 @@ follows:: /* Vector length vq * 16 bytes not supported */

.. [2] The maximum value vq for which the above condition is true is - max_vq. This is the maximum vector length available to the guest on - this vcpu, and determines which register slices are visible through - this ioctl interface. + max_vq. This is the maximum vector length currently available to + the guest on this vcpu, and determines which register slices are + visible through this ioctl interface. + + If SME is supported then the max_vq used for the Z and P registers + then while SVCR.SM is 1 this vector length will be the maximum SME + vector length available for the guest, otherwise it will be the + maximum SVE vector length available.

(See Documentation/arch/arm64/sve.rst for an explanation of the "vq" nomenclature.)

-KVM_REG_ARM64_SVE_VLS is only accessible after KVM_ARM_VCPU_INIT. -KVM_ARM_VCPU_INIT initialises it to the best set of vector lengths that -the host supports. +KVM_REG_ARM64_SVE_VLS and KVM_REG_ARM_SME_VLS are only accessible +after KVM_ARM_VCPU_INIT. KVM_ARM_VCPU_INIT initialises them to the +best set of vector lengths that the host supports.

-Userspace may subsequently modify it if desired until the vcpu's SVE -configuration is finalized using KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE). +Userspace may subsequently modify these registers if desired until the +vcpu's SVE and SME configuration is finalized using +KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC).

Apart from simply removing all vector lengths from the host set that exceed some value, support for arbitrarily chosen sets of vector lengths @@ -2685,8 +2701,8 @@ is hardware-dependent and may not be available. Attempting to configure an invalid set of vector lengths via KVM_SET_ONE_REG will fail with EINVAL.

-After the vcpu's SVE configuration is finalized, further attempts to -write this register will fail with EPERM. +After the vcpu's SVE or SME configuration is finalized, further +attempts to write these registers will fail with EPERM.

arm64 bitmap feature firmware pseudo-registers have the following bit pattern::

@@ -3469,6 +3485,7 @@ The initial values are defined as: - General Purpose registers, including PC and SP: set to 0 - FPSIMD/NEON registers: set to 0 - SVE registers: set to 0 + - SME registers: set to 0 - System registers: Reset to their architecturally defined values as for a warm reset to EL1 (resp. SVC) or EL2 (in the case of EL2 being enabled). @@ -3512,7 +3529,7 @@ Possible features:

- KVM_ARM_VCPU_SVE: Enables SVE for the CPU (arm64 only). Depends on KVM_CAP_ARM_SVE. - Requires KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE): + Requires KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC):

* After KVM_ARM_VCPU_INIT:

@@ -3520,7 +3537,7 @@ Possible features: initial value of this pseudo-register indicates the best set of vector lengths possible for a vcpu on this host.

- * Before KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE): + * Before KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC}):

- KVM_RUN and KVM_GET_REG_LIST are not available;

@@ -3533,11 +3550,40 @@ Possible features: KVM_SET_ONE_REG, to modify the set of vector lengths available for the vcpu.

- * After KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_SVE): + * After KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC):

- the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can no longer be written using KVM_SET_ONE_REG.

+ - KVM_ARM_VCPU_SME: Enables SME for the CPU (arm64 only). + Depends on KVM_CAP_ARM_SME. + Requires KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC): + + * After KVM_ARM_VCPU_INIT: + + - KVM_REG_ARM64_SME_VLS may be read using KVM_GET_ONE_REG: the + initial value of this pseudo-register indicates the best set of + vector lengths possible for a vcpu on this host. + + * Before KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC}): + + - KVM_RUN and KVM_GET_REG_LIST are not available; + + - KVM_GET_ONE_REG and KVM_SET_ONE_REG cannot be used to access + the scalable architectural SVE registers + KVM_REG_ARM64_SVE_ZREG(), KVM_REG_ARM64_SVE_PREG() or + KVM_REG_ARM64_SVE_FFR, the matrix register + KVM_REG_ARM64_SME_ZA() or the LUT register KVM_REG_ARM64_ZT(); + + - KVM_REG_ARM64_SME_VLS may optionally be written using + KVM_SET_ONE_REG, to modify the set of vector lengths available + for the vcpu. + + * After KVM_ARM_VCPU_FINALIZE(KVM_ARM_VCPU_VEC): + + - the KVM_REG_ARM64_SME_VLS pseudo-register is immutable, and can + no longer be written using KVM_SET_ONE_REG. + - KVM_ARM_VCPU_HAS_EL2: Enable Nested Virtualisation support, booting the guest from EL2 instead of EL1. Depends on KVM_CAP_ARM_EL2. @@ -5120,11 +5166,12 @@ Errors:

Recognised values for feature:

- ===== =========================================== - arm64 KVM_ARM_VCPU_SVE (requires KVM_CAP_ARM_SVE) - ===== =========================================== + ===== ============================================================== + arm64 KVM_ARM_VCPU_VEC (requires KVM_CAP_ARM_SVE or KVM_CAP_ARM_SME) + arm64 KVM_ARM_VCPU_SVE (alias for KVM_ARM_VCPU_VEC) + ===== ==============================================================

-Finalizes the configuration of the specified vcpu feature. +Finalizes the configuration of the specified vcpu features.

The vcpu must already have been initialised, enabling the affected feature, by means of a successful :ref:`KVM_ARM_VCPU_INIT <KVM_ARM_VCPU_INIT>` call with the

In order to simplify interdependencies in the rest of the series define the feature detection for SME and it's subfeatures. Due to the need for vector length configuration we define a flag for SME like for SVE. We also have two subfeatures which add architectural state, FA64 and SME2, which are configured via the normal ID register scheme.

Also provide helpers which check if the vCPU is in streaming mode or has ZA enabled.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 35 ++++++++++++++++++++++++++++++++++- arch/arm64/kvm/sys_regs.c | 2 +- 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 88d184aac789..98761b601457 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -355,6 +355,8 @@ struct kvm_arch { #define KVM_ARCH_FLAG_GUEST_HAS_SVE 9 /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ #define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 + /* SME exposed to guest */ +#define KVM_ARCH_FLAG_GUEST_HAS_SME 11 unsigned long flags;

/* VM-wide vCPU feature set */ @@ -1059,7 +1061,16 @@ struct kvm_vcpu_arch { #define vcpu_has_sve(vcpu) kvm_has_sve((vcpu)->kvm) #endif

-#define vcpu_has_vec(vcpu) vcpu_has_sve(vcpu) +#define kvm_has_sme(kvm) (system_supports_sme() && \ + test_bit(KVM_ARCH_FLAG_GUEST_HAS_SME, &(kvm)->arch.flags)) + +#ifdef __KVM_NVHE_HYPERVISOR__ +#define vcpu_has_sme(vcpu) kvm_has_sme(kern_hyp_va((vcpu)->kvm)) +#else +#define vcpu_has_sme(vcpu) kvm_has_sme((vcpu)->kvm) +#endif + +#define vcpu_has_vec(vcpu) (vcpu_has_sve(vcpu) || vcpu_has_sme(vcpu))

#ifdef CONFIG_ARM64_PTR_AUTH #define vcpu_has_ptrauth(vcpu) \ @@ -1705,6 +1716,28 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val); #define kvm_has_sctlr2(k) \ (kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))

+#define kvm_has_fa64(k) \ + (system_supports_fa64() && \ + kvm_has_feat((k), ID_AA64SMFR0_EL1, FA64, IMP)) + +#define kvm_has_sme2(k) \ + (system_supports_sme2() && \ + kvm_has_feat((k), ID_AA64PFR1_EL1, SME, SME2)) + +#ifdef __KVM_NVHE_HYPERVISOR__ +#define vcpu_has_sme2(vcpu) kvm_has_sme2(kern_hyp_va((vcpu)->kvm)) +#define vcpu_has_fa64(vcpu) kvm_has_fa64(kern_hyp_va((vcpu)->kvm)) +#else +#define vcpu_has_sme2(vcpu) kvm_has_sme2((vcpu)->kvm) +#define vcpu_has_fa64(vcpu) kvm_has_fa64((vcpu)->kvm) +#endif + +#define vcpu_in_streaming_mode(vcpu) \ + (__vcpu_sys_reg(vcpu, SVCR) & SVCR_SM_MASK) + +#define vcpu_za_enabled(vcpu) \ + (__vcpu_sys_reg(vcpu, SVCR) & SVCR_ZA_MASK) + static inline bool kvm_arch_has_irq_bypass(void) { return true; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 82ffb3b3b3cf..111b07a78787 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1782,7 +1782,7 @@ static unsigned int sve_visibility(const struct kvm_vcpu *vcpu, static unsigned int sme_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { - if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SME, IMP)) + if (vcpu_has_sme(vcpu)) return 0;

return REG_HIDDEN;

SME adds a second vector length configured in a very similar way to the SVE vector length, in order to facilitate future code sharing for SME refactor our storage of vector lengths to use an array like the host does. We do not yet take much advantage of this so the intermediate code is not as clean as might be.

No functional change.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 17 +++++++++++------ arch/arm64/include/asm/kvm_hyp.h | 2 +- arch/arm64/include/asm/kvm_pkvm.h | 2 +- arch/arm64/kvm/fpsimd.c | 2 +- arch/arm64/kvm/guest.c | 6 +++--- arch/arm64/kvm/hyp/include/hyp/switch.h | 6 +++--- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 6 +++--- arch/arm64/kvm/hyp/nvhe/pkvm.c | 7 ++++--- arch/arm64/kvm/reset.c | 22 +++++++++++----------- 9 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 98761b601457..59b88206d14c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -76,8 +76,10 @@ enum kvm_mode kvm_get_mode(void); static inline enum kvm_mode kvm_get_mode(void) { return KVM_MODE_NONE; }; #endif

-extern unsigned int __ro_after_init kvm_sve_max_vl; -extern unsigned int __ro_after_init kvm_host_sve_max_vl; +extern unsigned int __ro_after_init kvm_max_vl[ARM64_VEC_MAX]; +extern unsigned int __ro_after_init kvm_host_max_vl[ARM64_VEC_MAX]; +DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); + int __init kvm_arm_init_sve(void);

u32 __attribute_const__ kvm_target_cpu(void); @@ -813,7 +815,7 @@ struct kvm_vcpu_arch { */ void *sve_state; enum fp_type fp_type; - unsigned int sve_max_vl; + unsigned int max_vl[ARM64_VEC_MAX];

/* Stage 2 paging state used by the hardware on next switch */ struct kvm_s2_mmu *hw_mmu; @@ -1095,9 +1097,12 @@ struct kvm_vcpu_arch {

/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl)) + sve_ffr_offset((vcpu)->arch.max_vl[ARM64_VEC_SVE])) + +#define vcpu_vec_max_vq(vcpu, type) sve_vq_from_vl((vcpu)->arch.max_vl[type]) + +#define vcpu_sve_max_vq(vcpu) vcpu_vec_max_vq(vcpu, ARM64_VEC_SVE)

-#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)

#define vcpu_sve_zcr_elx(vcpu) \ (unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1) @@ -1116,7 +1121,7 @@ struct kvm_vcpu_arch { __size_ret; \ })

-#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.sve_max_vl) +#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.max_vl[ARM64_VEC_SVE])

/* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index e6be1f5d0967..0ad5a66e0d25 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -145,6 +145,6 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val);

extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); -extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); +extern unsigned int kvm_nvhe_sym(kvm_host_max_vl[ARM64_VEC_MAX]);

#endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index ea58282f59bb..6925606f2263 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -166,7 +166,7 @@ static inline size_t pkvm_host_sve_state_size(void) return 0;

return size_add(sizeof(struct cpu_sve_state), - SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl))); + SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]))); }

struct pkvm_mapping { diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 9158353d8be3..1f4fcc8b5554 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -75,7 +75,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) */ fp_state.st = &vcpu->arch.ctxt.fp_regs; fp_state.sve_state = vcpu->arch.sve_state; - fp_state.sve_vl = vcpu->arch.sve_max_vl; + fp_state.sve_vl = vcpu->arch.max_vl[ARM64_VEC_SVE]; fp_state.sme_state = NULL; fp_state.svcr = __ctxt_sys_reg(&vcpu->arch.ctxt, SVCR); fp_state.fpmr = __ctxt_sys_reg(&vcpu->arch.ctxt, FPMR); diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 6eb273b525d5..81f2b1111180 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -318,7 +318,7 @@ static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!vcpu_has_sve(vcpu)) return -ENOENT;

- if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) + if (WARN_ON(!sve_vl_valid(vcpu->arch.max_vl[ARM64_VEC_SVE]))) return -EINVAL;

memset(vqs, 0, sizeof(vqs)); @@ -356,7 +356,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (vq_present(vqs, vq)) max_vq = vq;

- if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) + if (max_vq > sve_vq_from_vl(kvm_max_vl[ARM64_VEC_SVE])) return -EINVAL;

/* @@ -375,7 +375,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return -EINVAL;

/* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ - vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); + vcpu->arch.max_vl[ARM64_VEC_SVE] = sve_vl_from_vq(max_vq);

return 0; } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index d5620e02787f..cd76129c6630 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -578,8 +578,8 @@ static inline void __hyp_sve_save_host(void) struct cpu_sve_state *sve_state = *host_data_ptr(sve_state);

sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); - write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), + write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2); + __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_max_vl[ARM64_VEC_SVE]), &sve_state->fpsr, true); } @@ -634,7 +634,7 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; write_sysreg_el2(zcr_el2, SYS_ZCR); } else { - zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + zcr_el2 = sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1; write_sysreg_el2(zcr_el2, SYS_ZCR);

zcr_el1 = vcpu_sve_max_vq(vcpu) - 1; diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 3206b2c07f82..76be13efcfcb 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -34,7 +34,7 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); - write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); + write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2); }

static void __hyp_sve_restore_host(void) @@ -50,8 +50,8 @@ static void __hyp_sve_restore_host(void) * that was discovered, if we wish to use larger VLs this will * need to be revisited. */ - write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), + write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2); + __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_max_vl[ARM64_VEC_SVE]), &sve_state->fpsr, true); write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index a461f192230a..65c49a5c7091 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -20,7 +20,7 @@ unsigned long __icache_flags; /* Used by kvm_get_vttbr(). */ unsigned int kvm_arm_vmid_bits;

-unsigned int kvm_host_sve_max_vl; +unsigned int kvm_host_max_vl[ARM64_VEC_MAX];

/* * The currently loaded hyp vCPU for each physical CPU. Used only when @@ -425,7 +425,8 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h }

/* Limit guest vector length to the maximum supported by the host. */ - sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); + sve_max_vl = min(READ_ONCE(host_vcpu->arch.max_vl[ARM64_VEC_SVE]), + kvm_host_max_vl[ARM64_VEC_SVE]); sve_state_size = sve_state_size_from_vl(sve_max_vl); sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state));

@@ -439,7 +440,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h goto err;

vcpu->arch.sve_state = sve_state; - vcpu->arch.sve_max_vl = sve_max_vl; + vcpu->arch.max_vl[ARM64_VEC_SVE] = sve_max_vl;

return 0; err: diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index f7c63e145d54..a8684a1346ec 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -32,7 +32,7 @@

/* Maximum phys_shift supported for any VM on this host */ static u32 __ro_after_init kvm_ipa_limit; -unsigned int __ro_after_init kvm_host_sve_max_vl; +unsigned int __ro_after_init kvm_host_max_vl[ARM64_VEC_MAX];

/* * ARMv8 Reset Values @@ -46,14 +46,14 @@ unsigned int __ro_after_init kvm_host_sve_max_vl; #define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ PSR_AA32_I_BIT | PSR_AA32_F_BIT)

-unsigned int __ro_after_init kvm_sve_max_vl; +unsigned int __ro_after_init kvm_max_vl[ARM64_VEC_MAX];

int __init kvm_arm_init_sve(void) { if (system_supports_sve()) { - kvm_sve_max_vl = sve_max_virtualisable_vl(); - kvm_host_sve_max_vl = sve_max_vl(); - kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl; + kvm_max_vl[ARM64_VEC_SVE] = sve_max_virtualisable_vl(); + kvm_host_max_vl[ARM64_VEC_SVE] = sve_max_vl(); + kvm_nvhe_sym(kvm_host_max_vl[ARM64_VEC_SVE]) = kvm_host_max_vl[ARM64_VEC_SVE];

/* * The get_sve_reg()/set_sve_reg() ioctl interface will need @@ -61,16 +61,16 @@ int __init kvm_arm_init_sve(void) * order to support vector lengths greater than * VL_ARCH_MAX: */ - if (WARN_ON(kvm_sve_max_vl > VL_ARCH_MAX)) - kvm_sve_max_vl = VL_ARCH_MAX; + if (WARN_ON(kvm_max_vl[ARM64_VEC_SVE] > VL_ARCH_MAX)) + kvm_max_vl[ARM64_VEC_SVE] = VL_ARCH_MAX;

/* * Don't even try to make use of vector lengths that * aren't available on all CPUs, for now: */ - if (kvm_sve_max_vl < sve_max_vl()) + if (kvm_max_vl[ARM64_VEC_SVE] < sve_max_vl()) pr_warn("KVM: SVE vector length for guests limited to %u bytes\n", - kvm_sve_max_vl); + kvm_max_vl[ARM64_VEC_SVE]); }

return 0; @@ -78,7 +78,7 @@ int __init kvm_arm_init_sve(void)

static void kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu) { - vcpu->arch.sve_max_vl = kvm_sve_max_vl; + vcpu->arch.max_vl[ARM64_VEC_SVE] = kvm_max_vl[ARM64_VEC_SVE];

/* * Userspace can still customize the vector lengths by writing @@ -99,7 +99,7 @@ static int kvm_vcpu_finalize_vec(struct kvm_vcpu *vcpu) size_t reg_sz; int ret;

- vl = vcpu->arch.sve_max_vl; + vl = vcpu->arch.max_vl[ARM64_VEC_SVE];

/* * Responsibility for these properties is shared between

SME is configured by the system registers SMCR_EL1 and SMCR_EL2, add definitions and userspace access for them. These control the SME vector length in a manner similar to that for SVE and also have feature enable bits for SME2 and FA64. A subsequent patch will add management of them for guests as part of the general floating point context switch, as is done for the equivalent SVE registers.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/include/asm/vncr_mapping.h | 1 + arch/arm64/kvm/sys_regs.c | 37 ++++++++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4348f297b5ae..edb2acdb4bd1 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -506,6 +506,7 @@ enum vcpu_sysreg { CPTR_EL2, /* Architectural Feature Trap Register (EL2) */ HACR_EL2, /* Hypervisor Auxiliary Control Register */ ZCR_EL2, /* SVE Control Register (EL2) */ + SMCR_EL2, /* SME Control Register (EL2) */ TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */ TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */ TCR_EL2, /* Translation Control Register (EL2) */ @@ -544,6 +545,7 @@ enum vcpu_sysreg { VNCR(ACTLR_EL1),/* Auxiliary Control Register */ VNCR(CPACR_EL1),/* Coprocessor Access Control */ VNCR(ZCR_EL1), /* SVE Control */ + VNCR(SMCR_EL1), /* SME Control */ VNCR(TTBR0_EL1),/* Translation Table Base Register 0 */ VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */ VNCR(TCR_EL1), /* Translation Control Register */ diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h index f6ec500ad3fa..5ab6a964bccf 100644 --- a/arch/arm64/include/asm/vncr_mapping.h +++ b/arch/arm64/include/asm/vncr_mapping.h @@ -44,6 +44,7 @@ #define VNCR_HDFGWTR_EL2 0x1D8 #define VNCR_ZCR_EL1 0x1E0 #define VNCR_HAFGRTR_EL2 0x1E8 +#define VNCR_SMCR_EL1 0x1F0 #define VNCR_TTBR0_EL1 0x200 #define VNCR_TTBR1_EL1 0x210 #define VNCR_FAR_EL1 0x220 diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 111b07a78787..81742e9237c0 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -142,6 +142,7 @@ static bool get_el2_to_el1_mapping(unsigned int reg, MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL ); MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL ); MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL ); + MAPPED_EL2_SYSREG(SMCR_EL2, SMCR_EL1, NULL ); MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL ); MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL ); default: @@ -2589,6 +2590,37 @@ static bool access_gic_elrsr(struct kvm_vcpu *vcpu, return true; }

+static unsigned int sme_el2_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + return __el2_visibility(vcpu, rd, sme_visibility); +} + +static bool access_smcr_el2(struct kvm_vcpu *vcpu, + struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + unsigned int vq; + u64 smcr; + + if (guest_hyp_sve_traps_enabled(vcpu)) { + kvm_inject_nested_sve_trap(vcpu); + return true; + } + + if (!p->is_write) { + p->regval = vcpu_read_sys_reg(vcpu, SMCR_EL2); + return true; + } + + smcr = p->regval; + vq = SYS_FIELD_GET(SMCR_ELx, LEN, smcr) + 1; + vq = min(vq, vcpu_sme_max_vq(vcpu)); + vcpu_write_sys_reg(vcpu, SYS_FIELD_PREP(SMCR_ELx, LEN, vq - 1), + SMCR_EL2); + return true; +} + static unsigned int s1poe_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { @@ -3035,7 +3067,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility }, { SYS_DESC(SYS_TRFCR_EL1), undef_access }, { SYS_DESC(SYS_SMPRI_EL1), undef_access }, - { SYS_DESC(SYS_SMCR_EL1), undef_access }, + { SYS_DESC(SYS_SMCR_EL1), NULL, reset_val, SMCR_EL1, 0, .visibility = sme_visibility }, { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 }, { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 }, { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 }, @@ -3391,6 +3423,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {

EL2_REG_VNCR(HCRX_EL2, reset_val, 0),

+ EL2_REG_FILTERED(SMCR_EL2, access_smcr_el2, reset_val, 0, + sme_el2_visibility), + EL2_REG(TTBR0_EL2, access_rw, reset_val, 0), EL2_REG(TTBR1_EL2, access_rw, reset_val, 0), EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),

The primary register for identifying SME is ID_AA64PFR1_EL1.SME. This is hidden from guests unless SME is enabled by the VMM. When it is visible it is writable and can be used to control the availability of SME2.

There is also a new register ID_AA64SMFR0_EL1 which we make writable, forcing it to all bits 0 if SME is disabled. This includes the field SMEver giving the SME version, userspace is responsible for ensuring the value is consistent with ID_AA64PFR1_EL1.SME. It also includes FA64, a separately enableable extension which provides the full FPSIMD and SVE instruction set including FFR in streaming mode. Userspace can control the availability of FA64 by writing to this field. The other features enumerated there only add new instructions, there are no architectural controls for these.

There is a further identification register SMIDR_EL1 which provides a basic description of the SME microarchitecture, in a manner similar to MIDR_EL1 for the PE. It also describes support for priority management and a basic affinity description for shared SME units, plus some RES0 space. We do not support priority management for guests so this is hidden from guests, along with any new fields.

As for MIDR_EL1 and REVIDR_EL1 we expose the implementer and revision information to guests with the raw value from the CPU we are running on, this may present issues for asymmetric systems or for migration as it does for the existing registers.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 3 +++ arch/arm64/kvm/config.c | 8 +------ arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 11 +++++++++ arch/arm64/kvm/hyp/nvhe/pkvm.c | 4 +++- arch/arm64/kvm/sys_regs.c | 37 ++++++++++++++++++++++++++---- 5 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 632c74397fa9..5225dd9b752c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -402,6 +402,7 @@ struct kvm_arch { u64 revidr_el1; u64 aidr_el1; u64 ctr_el0; + u64 smidr_el1;

/* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; @@ -1646,6 +1647,8 @@ static inline u64 *__vm_id_reg(struct kvm_arch *ka, u32 reg) return &ka->revidr_el1; case SYS_AIDR_EL1: return &ka->aidr_el1; + case SYS_SMIDR_EL1: + return &ka->smidr_el1; default: WARN_ON_ONCE(1); return NULL; diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index da66c4a14775..a08e89e210d8 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -208,14 +208,8 @@ static bool feat_anerr(struct kvm *kvm)

static bool feat_sme_smps(struct kvm *kvm) { - /* - * Revists this if KVM ever supports SME -- this really should - * look at the guest's view of SMIDR_EL1. Funnily enough, this - * is not captured in the JSON file, but only as a note in the - * ARM ARM. - */ return (kvm_has_feat(kvm, FEAT_SME) && - (read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)); + (kvm_read_vm_id_reg(kvm, SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)); }

static bool feat_spe_fds(struct kvm *kvm) diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index 8c3b3d6df99f..d921db152119 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -125,6 +125,17 @@ static inline u64 ctxt_midr_el1(struct kvm_cpu_context *ctxt) return kvm_read_vm_id_reg(kvm, SYS_MIDR_EL1); }

+static inline u64 ctxt_smidr_el1(struct kvm_cpu_context *ctxt) +{ + struct kvm *kvm = kern_hyp_va(ctxt_to_vcpu(ctxt)->kvm); + + if (!(ctxt_is_guest(ctxt) && + test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags))) + return read_sysreg_s(SYS_SMIDR_EL1); + + return kvm_read_vm_id_reg(kvm, SYS_SMIDR_EL1); +} + static inline void __sysreg_save_common_state(struct kvm_cpu_context *ctxt) { *ctxt_mdscr_el1(ctxt) = read_sysreg(mdscr_el1); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 65c49a5c7091..fc9fb2e693f6 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -338,8 +338,10 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc host_kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);

- if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) + if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) { hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; + hyp_vm->kvm.arch.smidr_el1 = host_kvm->arch.smidr_el1; + }

return; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4166d396450d..36ba1986e7a1 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1612,7 +1612,9 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE_frac); }

- val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME); + if (!vcpu_has_sme(vcpu)) + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_RNDR_trap); val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_NMI); val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS); @@ -1731,6 +1733,10 @@ static unsigned int id_visibility(const struct kvm_vcpu *vcpu, if (!vcpu_has_sve(vcpu)) return REG_RAZ; break; + case SYS_ID_AA64SMFR0_EL1: + if (!vcpu_has_sme(vcpu)) + return REG_RAZ; + break; }

return 0; @@ -1758,10 +1764,25 @@ static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,

/* cpufeature ID register access trap handlers */

+static bool hidden_id_reg(struct kvm_vcpu *vcpu, + struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + switch (reg_to_encoding(r)) { + case SYS_SMIDR_EL1: + return !vcpu_has_sme(vcpu); + default: + return false; + } +} + static bool access_id_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { + if (hidden_id_reg(vcpu, p, r)) + return bad_trap(vcpu, p, r, "write to hidden ID register"); + if (p->is_write) return write_to_read_only(vcpu, p, r);

@@ -2789,12 +2810,15 @@ static bool access_imp_id_reg(struct kvm_vcpu *vcpu, static u64 __ro_after_init boot_cpu_midr_val; static u64 __ro_after_init boot_cpu_revidr_val; static u64 __ro_after_init boot_cpu_aidr_val; +static u64 __ro_after_init boot_cpu_smidr_val;

static void init_imp_id_regs(void) { boot_cpu_midr_val = read_sysreg(midr_el1); boot_cpu_revidr_val = read_sysreg(revidr_el1); boot_cpu_aidr_val = read_sysreg(aidr_el1); + if (system_supports_sme()) + boot_cpu_smidr_val = read_sysreg_s(SYS_SMIDR_EL1); }

static u64 reset_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) @@ -2806,6 +2830,8 @@ static u64 reset_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) return boot_cpu_revidr_val; case SYS_AIDR_EL1: return boot_cpu_aidr_val; + case SYS_SMIDR_EL1: + return boot_cpu_smidr_val; default: KVM_BUG_ON(1, vcpu->kvm); return 0; @@ -2972,7 +2998,6 @@ static const struct sys_reg_desc sys_reg_descs[] = { ID_AA64PFR1_EL1_MTE_frac | ID_AA64PFR1_EL1_NMI | ID_AA64PFR1_EL1_RNDR_trap | - ID_AA64PFR1_EL1_SME | ID_AA64PFR1_EL1_RES0 | ID_AA64PFR1_EL1_MPAM_frac | ID_AA64PFR1_EL1_RAS_frac | @@ -2983,7 +3008,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { ID_AA64PFR2_EL1_MTESTOREONLY), ID_UNALLOCATED(4,3), ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0), - ID_HIDDEN(ID_AA64SMFR0_EL1), + ID_WRITABLE(ID_AA64SMFR0_EL1, ~ID_AA64SMFR0_EL1_RES0), ID_UNALLOCATED(4,6), ID_WRITABLE(ID_AA64FPFR0_EL1, ~ID_AA64FPFR0_EL1_RES0),

@@ -3185,7 +3210,11 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_CLIDR_EL1), access_clidr, reset_clidr, CLIDR_EL1, .set_user = set_clidr, .val = ~CLIDR_EL1_RES0 }, { SYS_DESC(SYS_CCSIDR2_EL1), undef_access }, - { SYS_DESC(SYS_SMIDR_EL1), undef_access }, + IMPLEMENTATION_ID(SMIDR_EL1, (SMIDR_EL1_NSMC | SMIDR_EL1_HIP | + SMIDR_EL1_AFFINITY2 | + SMIDR_EL1_IMPLEMENTER | + SMIDR_EL1_REVISION | SMIDR_EL1_SH | + SMIDR_EL1_AFFINITY)), IMPLEMENTATION_ID(AIDR_EL1, GENMASK_ULL(63, 0)), { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 }, ID_FILTERED(CTR_EL0, ctr_el0,

Provide versions of the SME state save and restore functions for the hypervisor to allow it to restore ZA and ZT for guests.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_hyp.h | 3 +++ arch/arm64/kvm/hyp/fpsimd.S | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+)

diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 0ad5a66e0d25..1c31d8b26aa9 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -115,6 +115,9 @@ void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr); void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr); +int __sve_get_vl(void); +void __sme_save_state(void const *state, bool restore_zt); +void __sme_restore_state(void const *state, bool restore_zt);

u64 __guest_enter(struct kvm_vcpu *vcpu);

diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 6e16cbfc5df2..44a1b0a483da 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -29,3 +29,29 @@ SYM_FUNC_START(__sve_save_state) sve_save 0, x1, x2, 3 ret SYM_FUNC_END(__sve_save_state) + +SYM_FUNC_START(__sve_get_vl) + _sve_rdvl 0, 1 + ret +SYM_FUNC_END(__sve_get_vl) + +SYM_FUNC_START(__sme_save_state) + _sme_rdsvl 2, 1 // x2 = VL/8 + sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA + + cbz x1, 1f + _str_zt 0 +1: + ret +SYM_FUNC_END(__sme_save_state) + +SYM_FUNC_START(__sme_restore_state) + _sme_rdsvl 2, 1 // x2 = VL/8 + sme_load_za 0, x2, 12 // Leaves x0 pointing to end of ZA + + cbz x1, 1f + _ldr_zt 0 + +1: + ret +SYM_FUNC_END(__sme_restore_state)

Writes to the physical SVCR.SM and SVCR.ZA change the state of PSTATE.SM and PSTATE.ZA, causing other floating point state to reset. Emulate this behaviour for writes done via the KVM userspace ABI.

Setting PSTATE.ZA to 1 causes ZA and ZT0 to be reset to 0, these are stored in sme_state. Setting PSTATE.ZA to 0 causes ZA and ZT0 to become inaccesible so no reset is needed.

Any change in PSTATE.SM causes the V, Z, P, FFR and FPMR registers to be reset to 0 and FPSR to be reset to 0x800009f.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/kvm_host.h | 24 ++++++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 29 ++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ebc516fe09db..8e5c16372b83 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1144,6 +1144,30 @@ struct kvm_vcpu_arch {

#define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.max_vl[ARM64_VEC_SVE])

+#define vcpu_sme_state(vcpu) (kern_hyp_va((vcpu)->arch.sme_state)) + +#define sme_state_size_from_vl(vl, sme2) ({ \ + size_t __size_ret; \ + unsigned int __vq; \ + \ + if (WARN_ON(!sve_vl_valid(vl))) { \ + __size_ret = 0; \ + } else { \ + __vq = sve_vq_from_vl(vl); \ + __size_ret = ZA_SIG_REGS_SIZE(__vq); \ + if (sme2) \ + __size_ret += ZT_SIG_REG_SIZE; \ + } \ + \ + __size_ret; \ +}) + +#define vcpu_sme_state_size(vcpu) ({ \ + unsigned long __vl; \ + __vl = (vcpu)->arch.max_vl[ARM64_VEC_SME]; \ + sme_state_size_from_vl(__vl, vcpu_has_sme2(vcpu)); \ +}) + /* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the * memory backed version of a register, and not the one most recently diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7284705a5f0a..0752e119c056 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -788,6 +788,33 @@ static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; }

+static int set_svcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + u64 val) +{ + u64 old = __vcpu_sys_reg(vcpu, rd->reg); + + if (val & SVCR_RES0) + return -EINVAL; + + if ((val & SVCR_ZA) && !(old & SVCR_ZA) && vcpu->arch.sme_state) + memset(vcpu->arch.sme_state, 0, vcpu_sme_state_size(vcpu)); + + if ((val & SVCR_SM) != (old & SVCR_SM)) { + memset(vcpu->arch.ctxt.fp_regs.vregs, 0, + sizeof(vcpu->arch.ctxt.fp_regs.vregs)); + + if (vcpu->arch.sve_state) + memset(vcpu->arch.sve_state, 0, + vcpu_sve_state_size(vcpu)); + + __vcpu_assign_sys_reg(vcpu, FPMR, 0); + vcpu->arch.ctxt.fp_regs.fpsr = 0x800009f; + } + + __vcpu_assign_sys_reg(vcpu, rd->reg, val); + return 0; +} + static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { @@ -3248,7 +3275,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { CTR_EL0_DminLine_MASK | CTR_EL0_L1Ip_MASK | CTR_EL0_IminLine_MASK), - { SYS_DESC(SYS_SVCR), undef_access, reset_val, SVCR, 0, .visibility = sme_visibility }, + { SYS_DESC(SYS_SVCR), undef_access, reset_val, SVCR, 0, .visibility = sme_visibility, .set_user = set_svcr }, { SYS_DESC(SYS_FPMR), undef_access, reset_val, FPMR, 0, .visibility = fp8_visibility },

{ PMU_SYS_REG(PMCR_EL0), .access = access_pmcr, .reset = reset_pmcr,

If the guest has SME state we need to context switch that state, provide support for that for normal guests.

SME has three sets of registers, ZA, ZT (only present for SME2) and also streaming SVE which replaces the standard floating point registers when active. The first two are fairly straightforward, they are accessible only when PSTATE.ZA is set and we can reuse the assembly from the host to save and load them from a single contiguous buffer. When PSTATE.ZA is not set then these registers are inaccessible, when the guest enables PSTATE.ZA all bits will be set to 0 by that and nothing is required on restore.

Streaming mode is slightly more complicated, when enabled via PSTATE.SM it provides a version of the SVE registers using the SME vector length and may optionally omit the FFR register. SME may also be present without SVE. The register state is stored in sve_state as for non-streaming SVE mode, we make an initial selection of registers to update based on the guest SVE support and then override this when loading SVCR if streaming mode is enabled.

A further complication is that when the hardware is in streaming mode guest operations that are invalid in in streaming mode will generate SME exceptions. There are also subfeature exceptions for SME2 controlled via SMCR which generate distinct exception codes. In many situations these exceptions are routed directly to the lower ELs with no opportunity for the hypervisor to intercept. So that guests do not see unexpected exception types due to the actual hardware configuration not being what the guest configured we update the SMCRs and SVCR even if the guest does not own the registers.

Since in order to avoid duplication with SME we now restore the register state outside of the SVE specific restore function we need to move the restore of the effective VL for nested guests to a separate restore function run after loading the floating point register state, along with the similar handling required for SME.

The selection of which vector length to use is handled by vcpu_sve_pffr().

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/include/asm/fpsimd.h | 10 +++ arch/arm64/include/asm/kvm_emulate.h | 6 ++ arch/arm64/include/asm/kvm_host.h | 4 + arch/arm64/kvm/fpsimd.c | 25 ++++-- arch/arm64/kvm/hyp/include/hyp/switch.h | 151 ++++++++++++++++++++++++++++++-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 80 +++++++++++++++-- 6 files changed, 255 insertions(+), 21 deletions(-)

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 38c24c6485ad..40b56fba9c54 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -442,6 +442,15 @@ static inline size_t sme_state_size(struct task_struct const *task) write_sysreg_s(__new, (reg)); \ } while (0)

+#define sme_cond_update_smcr_vq(val, reg) \ + do { \ + u64 __smcr = read_sysreg_s((reg)); \ + u64 __new = __smcr & ~SMCR_ELx_LEN_MASK; \ + __new |= (val) & SMCR_ELx_LEN_MASK; \ + if (__smcr != __new) \ + write_sysreg_s(__new, (reg)); \ + } while (0) + #else

static inline void sme_user_disable(void) { BUILD_BUG(); } @@ -471,6 +480,7 @@ static inline size_t sme_state_size(struct task_struct const *task) }

#define sme_cond_update_smcr(val, fa64, zt0, reg) do { } while (0) +#define sme_cond_update_smcr_vq(val, reg) do { } while (0)

#endif /* ! CONFIG_ARM64_SME */

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index fa8a08a1ccd5..d71ccea726f2 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -674,4 +674,10 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En; } } + +static inline bool guest_hyp_sme_traps_enabled(const struct kvm_vcpu *vcpu) +{ + return __guest_hyp_cptr_xen_trap_enabled(vcpu, SMEN); +} + #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 8e5c16372b83..fa646da80e3e 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -738,6 +738,7 @@ struct kvm_host_data {

/* Used by pKVM only. */ u64 fpmr; + u64 smcr_el1;

/* Ownership of the FP regs */ enum { @@ -1128,6 +1129,9 @@ struct kvm_vcpu_arch { #define vcpu_sve_zcr_elx(vcpu) \ (unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1)

+#define vcpu_sme_smcr_elx(vcpu) \ + (unlikely(is_hyp_ctxt(vcpu)) ? SMCR_EL2 : SMCR_EL1) + #define sve_state_size_from_vl(sve_max_vl) ({ \ size_t __size_ret; \ unsigned int __vq; \ diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 1f4fcc8b5554..8fb8c55e50b3 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -69,19 +69,25 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) WARN_ON_ONCE(!irqs_disabled());

if (guest_owns_fp_regs()) { - /* - * Currently we do not support SME guests so SVCR is - * always 0 and we just need a variable to point to. - */ fp_state.st = &vcpu->arch.ctxt.fp_regs; fp_state.sve_state = vcpu->arch.sve_state; fp_state.sve_vl = vcpu->arch.max_vl[ARM64_VEC_SVE]; - fp_state.sme_state = NULL; + fp_state.sme_state = vcpu->arch.sme_state; + fp_state.sme_vl = vcpu->arch.max_vl[ARM64_VEC_SME]; fp_state.svcr = __ctxt_sys_reg(&vcpu->arch.ctxt, SVCR); fp_state.fpmr = __ctxt_sys_reg(&vcpu->arch.ctxt, FPMR); fp_state.fp_type = &vcpu->arch.fp_type; + fp_state.sme_features = 0; + if (kvm_has_fa64(vcpu->kvm)) + fp_state.sme_features |= SMCR_ELx_FA64; + if (kvm_has_sme2(vcpu->kvm)) + fp_state.sme_features |= SMCR_ELx_EZT0;

+ /* + * For SME only hosts fpsimd_save() will override the + * state selection if we are in streaming mode. + */ if (vcpu_has_sve(vcpu)) fp_state.to_save = FP_STATE_SVE; else @@ -90,6 +96,15 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) fpsimd_bind_state_to_cpu(&fp_state);

clear_thread_flag(TIF_FOREIGN_FPSTATE); + } else { + /* + * We might have enabled SME to configure traps but + * insist the host doesn't run the hypervisor with SME + * enabled, ensure it's disabled again. + */ + if (system_supports_sme()) { + sme_smstop(); + } } }

diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index cd76129c6630..c128b4d25a2d 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -551,6 +551,29 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) return true; }

+static inline void __hyp_sme_restore_guest(struct kvm_vcpu *vcpu, + bool *restore_sve, + bool *restore_ffr) +{ + bool has_fa64 = vcpu_has_fa64(vcpu); + bool has_sme2 = vcpu_has_sme2(vcpu); + + sme_cond_update_smcr(vcpu_sme_max_vq(vcpu) - 1, has_fa64, has_sme2, + SYS_SMCR_EL2); + + write_sysreg_el1(__vcpu_sys_reg(vcpu, SMCR_EL1), SYS_SMCR); + + write_sysreg_s(__vcpu_sys_reg(vcpu, SVCR), SYS_SVCR); + + if (vcpu_in_streaming_mode(vcpu)) { + *restore_sve = true; + *restore_ffr = has_fa64; + } + + if (vcpu_za_enabled(vcpu)) + __sme_restore_state(vcpu_sme_state(vcpu), has_sme2); +} + static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) { /* @@ -558,19 +581,25 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) * vCPU. Start off with the max VL so we can load the SVE state. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_restore_state(vcpu_sve_pffr(vcpu), - &vcpu->arch.ctxt.fp_regs.fpsr, - true);

+ write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR); +} + +static inline void __hyp_nv_restore_guest_vls(struct kvm_vcpu *vcpu) +{ /* * The effective VL for a VM could differ from the max VL when running a * nested guest, as the guest hypervisor could select a smaller VL. Slap * that into hardware before wrapping up. */ - if (is_nested_ctxt(vcpu)) + if (!is_nested_ctxt(vcpu)) + return; + + if (vcpu_has_sve(vcpu)) sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);

- write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR); + if (vcpu_has_sme(vcpu)) + sme_cond_update_smcr_vq(__vcpu_sys_reg(vcpu, SMCR_EL2), SYS_SMCR_EL2); }

static inline void __hyp_sve_save_host(void) @@ -584,10 +613,40 @@ static inline void __hyp_sve_save_host(void) true); }

+static inline void kvm_sme_configure_traps(struct kvm_vcpu *vcpu) +{ + u64 smcr_el1, smcr_el2; + u64 svcr; + + if (!vcpu_has_sme(vcpu)) + return; + + /* A guest hypervisor may restrict the effective max VL. */ + if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) + smcr_el2 = __vcpu_sys_reg(vcpu, SMCR_EL2); + else + smcr_el2 = vcpu_sme_max_vq(vcpu) - 1; + + if (vcpu_has_fa64(vcpu)) + smcr_el2 |= SMCR_ELx_FA64; + if (vcpu_has_sme2(vcpu)) + smcr_el2 |= SMCR_ELx_EZT0; + + write_sysreg_el2(smcr_el2, SYS_SMCR); + + smcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sme_smcr_elx(vcpu)); + write_sysreg_el1(smcr_el1, SYS_SMCR); + + svcr = __vcpu_sys_reg(vcpu, SVCR); + write_sysreg_s(svcr, SYS_SVCR); +} + static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) { u64 zcr_el1, zcr_el2;

+ kvm_sme_configure_traps(vcpu); + if (!guest_owns_fp_regs()) return;

@@ -607,8 +666,50 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)

static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) { + u64 smcr_el1, smcr_el2; u64 zcr_el1, zcr_el2;

+ if (vcpu_has_sme(vcpu)) { + /* + * __deactivate_cptr_traps( disabled traps), but there + * hasn't necessarily been a context synchronization + * event yet. + */ + isb(); + + smcr_el1 = read_sysreg_el1(SYS_SMCR); + __vcpu_assign_sys_reg(vcpu, vcpu_sme_smcr_elx(vcpu), smcr_el1); + + smcr_el2 = 0; + if (system_supports_fa64()) + smcr_el2 |= SMCR_ELx_FA64; + if (system_supports_sme2()) + smcr_el2 |= SMCR_ELx_EZT0; + + /* + * The guest's state is always saved using the guest's max VL. + * Ensure that the host has the guest's max VL active such that + * the host can save the guest's state lazily, but don't + * artificially restrict the host to the guest's max VL. + */ + if (has_vhe()) { + smcr_el2 |= vcpu_sme_max_vq(vcpu) - 1; + write_sysreg_el2(smcr_el2, SYS_SMCR); + } else { + smcr_el1 = smcr_el2; + smcr_el2 |= sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SME]) - 1; + write_sysreg_el2(smcr_el2, SYS_SMCR); + + smcr_el1 |= vcpu_sve_max_vq(vcpu) - 1; + write_sysreg_el1(smcr_el1, SYS_SMCR); + } + + if (guest_owns_fp_regs()) { + u64 svcr = read_sysreg_s(SYS_SVCR); + __vcpu_assign_sys_reg(vcpu, SVCR, svcr); + } + } + if (!guest_owns_fp_regs()) return;

@@ -645,6 +746,16 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)

static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) { + /* + * The hypervisor refuses to run if streaming mode or ZA is + * enabled, we only need to save SMCR_EL1 for SME. For pKVM + * we will restore this, reset SMCR_EL2 to a fixed value and + * disable streaming mode and ZA to avoid any state being + * leaked. + */ + if (system_supports_sme()) + *host_data_ptr(smcr_el1) = read_sysreg_el1(SYS_SMCR); + /* * Non-protected kvm relies on the host restoring its sve state. * Protected kvm restores the host's sve state as not to reveal that @@ -669,14 +780,17 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) */ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) { - bool sve_guest; - u8 esr_ec; + bool restore_sve, restore_ffr; + bool sve_guest, sme_guest; + u8 esr_ec, esr_iss_smtc;

if (!system_supports_fpsimd()) return false;

sve_guest = vcpu_has_sve(vcpu); + sme_guest = vcpu_has_sme(vcpu); esr_ec = kvm_vcpu_trap_get_class(vcpu); + esr_iss_smtc = ESR_ELx_SME_ISS_SMTC((kvm_vcpu_get_esr(vcpu)));

/* Only handle traps the vCPU can support here: */ switch (esr_ec) { @@ -695,6 +809,15 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) if (guest_hyp_sve_traps_enabled(vcpu)) return false; break; + case ESR_ELx_EC_SME: + if (!sme_guest) + return false; + if (guest_hyp_sme_traps_enabled(vcpu)) + return false; + if (!kvm_has_sme2(vcpu->kvm) && + (esr_iss_smtc == ESR_ELx_SME_ISS_SMTC_ZT_DISABLED)) + return false; + break; default: return false; } @@ -710,8 +833,20 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) kvm_hyp_save_fpsimd_host(vcpu);

/* Restore the guest state */ + + /* These may be overridden for a SME guest */ + restore_sve = sve_guest; + restore_ffr = sve_guest; + if (sve_guest) __hyp_sve_restore_guest(vcpu); + if (sme_guest) + __hyp_sme_restore_guest(vcpu, &restore_sve, &restore_ffr); + + if (restore_sve) + __sve_restore_state(vcpu_sve_pffr(vcpu), + &vcpu->arch.ctxt.fp_regs.fpsr, + restore_ffr); else __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);

@@ -722,6 +857,8 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) if (!(read_sysreg(hcr_el2) & HCR_RW)) write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2);

+ __hyp_nv_restore_guest_vls(vcpu); + *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED;

/* diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 76be13efcfcb..ccbe0389c0b7 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -26,14 +26,17 @@ void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);

static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) { + bool save_ffr = !vcpu_in_streaming_mode(vcpu) || vcpu_has_fa64(vcpu); + __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); + /* * On saving/restoring guest sve state, always use the maximum VL for * the guest. The layout of the data when saving the sve state depends * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); + __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, save_ffr); write_sysreg_s(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SVE]) - 1, SYS_ZCR_EL2); }

@@ -57,9 +60,63 @@ static void __hyp_sve_restore_host(void) write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); }

-static void fpsimd_sve_flush(void) +static void __hyp_sme_save_guest(struct kvm_vcpu *vcpu) { - *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; + __vcpu_assign_sys_reg(vcpu, SMCR_EL1, read_sysreg_el1(SYS_SMCR)); + __vcpu_assign_sys_reg(vcpu, SVCR, read_sysreg_s(SYS_SVCR)); + + /* + * On saving/restoring guest sve state, always use the maximum VL for + * the guest. The layout of the data when saving the sve state depends + * on the VL, so use a consistent (i.e., the maximum) guest VL. + * + * We restore the FA64 and SME2 enables for the host since we + * will always restore the host configuration so if host and + * guest VLs are the same we might suppress an update. + */ + sme_cond_update_smcr(vcpu_sme_max_vq(vcpu) - 1, system_supports_fa64(), + system_supports_sme2(), SYS_SMCR_EL2); + + if (vcpu_za_enabled(vcpu)) + __sme_save_state(vcpu_sme_state(vcpu), vcpu_has_sme2(vcpu)); +} + +static void __hyp_sme_restore_host(void) +{ + /* + * The hypervisor refuses to run if we are in streaming mode + * or have ZA enabled so there is no SME specific state to + * restore other than the system registers. + * + * Note that this constrains the PE to the maximum shared VL + * that was discovered, if we wish to use larger VLs this will + * need to be revisited. + */ + sme_cond_update_smcr(sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SME]) - 1, + cpus_have_final_cap(ARM64_SME_FA64), + cpus_have_final_cap(ARM64_SME2), SYS_SMCR_EL2); + + write_sysreg_el1(*host_data_ptr(smcr_el1), SYS_SMCR); + + sme_smstop(); +} + +static void fpsimd_sve_flush(struct kvm_vcpu *vcpu) +{ + /* + * If the guest has SME then we need to restore the trap + * controls in SMCR and mode in SVCR in order to ensure that + * traps generated directly to EL1 have the correct types, + * otherwise we can defer until we load the guest state. + */ + if (vcpu_has_sme(vcpu)) { + kvm_hyp_save_fpsimd_host(vcpu); + kvm_sme_configure_traps(vcpu); + + *host_data_ptr(fp_owner) = FP_STATE_FREE; + } else { + *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; + } }

static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) @@ -75,7 +132,10 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) */ isb();

- if (vcpu_has_sve(vcpu)) + if (vcpu_has_sme(vcpu)) + __hyp_sme_save_guest(vcpu); + + if (vcpu_has_sve(vcpu) || vcpu_in_streaming_mode(vcpu)) __hyp_sve_save_guest(vcpu); else __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); @@ -84,6 +144,9 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (has_fpmr) __vcpu_assign_sys_reg(vcpu, FPMR, read_sysreg_s(SYS_FPMR));

+ if (system_supports_sme()) + __hyp_sme_restore_host(); + if (system_supports_sve()) __hyp_sve_restore_host(); else @@ -121,7 +184,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;

- fpsimd_sve_flush(); + fpsimd_sve_flush(host_vcpu); flush_debug_state(hyp_vcpu);

hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; @@ -203,10 +266,9 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();

/* - * KVM (and pKVM) doesn't support SME guests for now, and - * ensures that SME features aren't enabled in pstate when - * loading a vcpu. Therefore, if SME features enabled the host - * is misbehaving. + * KVM (and pKVM) refuses to run if PSTATE.{SM,ZA} are + * enabled. Therefore, if SME features enabled the + * host is misbehaving. */ if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { ret = -EINVAL;

With support for context switching SME state in place allow access to SME in nested guests.

The SME floating point state is handled along with all the other floating point state, SME specific floating point exceptions are directed into the same handlers as other floating point exceptions with NV specific handling for the vector lengths already in place.

TPIDR2_EL0 is context switched along with the other TPIDRs as part of the main guest register context switch.

SME priority support is currently masked from all guests including nested ones.

Signed-off-by: Mark Brown broonie@kernel.org --- arch/arm64/kvm/nested.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 153b3e11b115..804694f2dc9a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1458,9 +1458,10 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) break;

case SYS_ID_AA64PFR1_EL1: - /* Only support BTI, SSBS, CSV2_frac */ + /* Only support BTI, SSBS, SME, CSV2_frac */ val &= (ID_AA64PFR1_EL1_BT | ID_AA64PFR1_EL1_SSBS | + ID_AA64PFR1_EL1_SME | ID_AA64PFR1_EL1_CSV2_frac); break;

SME adds a number of new system registers, update get-reg-list to check for them based on the visibility of SME.

Signed-off-by: Mark Brown broonie@kernel.org --- tools/testing/selftests/kvm/arm64/get-reg-list.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index 011fad95dd02..54b371330f57 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -61,7 +61,13 @@ static struct feature_id_reg feat_id_regs[] = { REG_FEAT(HFGITR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2), REG_FEAT(HDFGRTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2), REG_FEAT(HDFGWTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2), - REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP), + REG_FEAT(SMCR_EL1, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(SMCR_EL2, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(SMIDR_EL1, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(SMPRI_EL1, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(SMPRIMAP_EL2, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(TPIDR2_EL0, ID_AA64PFR1_EL1, SME, IMP), + REG_FEAT(SVCR, ID_AA64PFR1_EL1, SME, IMP), REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP), REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP), REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP), @@ -351,6 +357,7 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 0, 0, 0, 0), /* MIDR_EL1 */ ARM64_SYS_REG(3, 0, 0, 0, 6), /* REVIDR_EL1 */ ARM64_SYS_REG(3, 1, 0, 0, 1), /* CLIDR_EL1 */ + ARM64_SYS_REG(3, 1, 0, 0, 6), /* SMIDR_EL1 */ ARM64_SYS_REG(3, 1, 0, 0, 7), /* AIDR_EL1 */ ARM64_SYS_REG(3, 3, 0, 0, 1), /* CTR_EL0 */ ARM64_SYS_REG(2, 0, 0, 0, 4), @@ -482,6 +489,8 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 0, 1, 0, 1), /* ACTLR_EL1 */ ARM64_SYS_REG(3, 0, 1, 0, 2), /* CPACR_EL1 */ KVM_ARM64_SYS_REG(SYS_SCTLR2_EL1), + ARM64_SYS_REG(3, 0, 1, 2, 4), /* SMPRI_EL1 */ + ARM64_SYS_REG(3, 0, 1, 2, 6), /* SMCR_EL1 */ ARM64_SYS_REG(3, 0, 2, 0, 0), /* TTBR0_EL1 */ ARM64_SYS_REG(3, 0, 2, 0, 1), /* TTBR1_EL1 */ ARM64_SYS_REG(3, 0, 2, 0, 2), /* TCR_EL1 */ @@ -502,9 +511,11 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 0, 13, 0, 4), /* TPIDR_EL1 */ ARM64_SYS_REG(3, 0, 14, 1, 0), /* CNTKCTL_EL1 */ ARM64_SYS_REG(3, 2, 0, 0, 0), /* CSSELR_EL1 */ + ARM64_SYS_REG(3, 3, 4, 2, 2), /* SVCR */ ARM64_SYS_REG(3, 3, 10, 2, 4), /* POR_EL0 */ ARM64_SYS_REG(3, 3, 13, 0, 2), /* TPIDR_EL0 */ ARM64_SYS_REG(3, 3, 13, 0, 3), /* TPIDRRO_EL0 */ + ARM64_SYS_REG(3, 3, 13, 0, 5), /* TPIDR2_EL0 */ ARM64_SYS_REG(3, 3, 14, 0, 1), /* CNTPCT_EL0 */ ARM64_SYS_REG(3, 3, 14, 2, 1), /* CNTP_CTL_EL0 */ ARM64_SYS_REG(3, 3, 14, 2, 2), /* CNTP_CVAL_EL0 */ @@ -713,6 +724,8 @@ static __u64 el2_regs[] = { SYS_REG(HFGITR_EL2), SYS_REG(HACR_EL2), SYS_REG(ZCR_EL2), + SYS_REG(SMPRIMAP_EL2), + SYS_REG(SMCR_EL2), SYS_REG(HCRX_EL2), SYS_REG(TTBR0_EL2), SYS_REG(TTBR1_EL2),