Skip to content

Commit fed55f4

Browse files
D Scott PhillipsMarc Zyngier
D Scott Phillips
authored and
Marc Zyngier
committed
arm64: errata: Work around AmpereOne's erratum AC04_CPU_23
On AmpereOne AC04, updates to HCR_EL2 can rarely corrupt simultaneous translations for data addresses initiated by load/store instructions. Only instruction initiated translations are vulnerable, not translations from prefetches for example. A DSB before the store to HCR_EL2 is sufficient to prevent older instructions from hitting the window for corruption, and an ISB after is sufficient to prevent younger instructions from hitting the window for corruption. Signed-off-by: D Scott Phillips <scott@os.amperecomputing.com> Reviewed-by: Oliver Upton <oliver.upton@linux.dev> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Link: https://lore.kernel.org/r/20250513184514.2678288-1-scott@os.amperecomputing.com Signed-off-by: Marc Zyngier <maz@kernel.org>
1 parent 92c749e commit fed55f4

File tree

17 files changed

+80
-19
lines changed

17 files changed

+80
-19
lines changed

‎Documentation/arch/arm64/silicon-errata.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ stable kernels.
5757
+----------------+-----------------+-----------------+-----------------------------+
5858
| Ampere | AmpereOne AC04 | AC04_CPU_10 | AMPERE_ERRATUM_AC03_CPU_38 |
5959
+----------------+-----------------+-----------------+-----------------------------+
60+
| Ampere | AmpereOne AC04 | AC04_CPU_23 | AMPERE_ERRATUM_AC04_CPU_23 |
61+
+----------------+-----------------+-----------------+-----------------------------+
6062
+----------------+-----------------+-----------------+-----------------------------+
6163
| ARM | Cortex-A510 | #2457168 | ARM64_ERRATUM_2457168 |
6264
+----------------+-----------------+-----------------+-----------------------------+

‎arch/arm64/Kconfig

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,23 @@ config AMPERE_ERRATUM_AC03_CPU_38
464464

465465
If unsure, say Y.
466466

467+
config AMPERE_ERRATUM_AC04_CPU_23
468+
bool "AmpereOne: AC04_CPU_23: Failure to synchronize writes to HCR_EL2 may corrupt address translations."
469+
default y
470+
help
471+
This option adds an alternative code sequence to work around Ampere
472+
errata AC04_CPU_23 on AmpereOne.
473+
474+
Updates to HCR_EL2 can rarely corrupt simultaneous translations for
475+
data addresses initiated by load/store instructions. Only
476+
instruction initiated translations are vulnerable, not translations
477+
from prefetches for example. A DSB before the store to HCR_EL2 is
478+
sufficient to prevent older instructions from hitting the window
479+
for corruption, and an ISB after is sufficient to prevent younger
480+
instructions from hitting the window for corruption.
481+
482+
If unsure, say Y.
483+
467484
config ARM64_WORKAROUND_CLEAN_CACHE
468485
bool
469486

‎arch/arm64/include/asm/el2_setup.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838

3939
orr x0, x0, #HCR_E2H
4040
.LnVHE_\@:
41-
msr hcr_el2, x0
41+
msr_hcr_el2 x0
4242
isb
4343
.endm
4444

‎arch/arm64/include/asm/hardirq.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ do { \
4141
\
4242
___hcr = read_sysreg(hcr_el2); \
4343
if (!(___hcr & HCR_TGE)) { \
44-
write_sysreg(___hcr | HCR_TGE, hcr_el2); \
44+
write_sysreg_hcr(___hcr | HCR_TGE); \
4545
isb(); \
4646
} \
4747
/* \
@@ -82,7 +82,7 @@ do { \
8282
*/ \
8383
barrier(); \
8484
if (!___ctx->cnt && !(___hcr & HCR_TGE)) \
85-
write_sysreg(___hcr, hcr_el2); \
85+
write_sysreg_hcr(___hcr); \
8686
} while (0)
8787

8888
static inline void ack_bad_irq(unsigned int irq)

‎arch/arm64/include/asm/sysreg.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,6 +1091,15 @@
10911091
__emit_inst(0xd5000000|(\sreg)|(.L__gpr_num_\rt))
10921092
.endm
10931093

1094+
.macro msr_hcr_el2, reg
1095+
#if IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23)
1096+
dsb nsh
1097+
msr hcr_el2, \reg
1098+
isb
1099+
#else
1100+
msr hcr_el2, \reg
1101+
#endif
1102+
.endm
10941103
#else
10951104

10961105
#include <linux/bitfield.h>
@@ -1178,13 +1187,31 @@
11781187
write_sysreg(__scs_new, sysreg); \
11791188
} while (0)
11801189

1190+
#define sysreg_clear_set_hcr(clear, set) do { \
1191+
u64 __scs_val = read_sysreg(hcr_el2); \
1192+
u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \
1193+
if (__scs_new != __scs_val) \
1194+
write_sysreg_hcr(__scs_new); \
1195+
} while (0)
1196+
11811197
#define sysreg_clear_set_s(sysreg, clear, set) do { \
11821198
u64 __scs_val = read_sysreg_s(sysreg); \
11831199
u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \
11841200
if (__scs_new != __scs_val) \
11851201
write_sysreg_s(__scs_new, sysreg); \
11861202
} while (0)
11871203

1204+
#define write_sysreg_hcr(__val) do { \
1205+
if (IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23) && \
1206+
(!system_capabilities_finalized() || \
1207+
alternative_has_cap_unlikely(ARM64_WORKAROUND_AMPERE_AC04_CPU_23))) \
1208+
asm volatile("dsb nsh; msr hcr_el2, %x0; isb" \
1209+
: : "rZ" (__val)); \
1210+
else \
1211+
asm volatile("msr hcr_el2, %x0" \
1212+
: : "rZ" (__val)); \
1213+
} while (0)
1214+
11881215
#define read_sysreg_par() ({ \
11891216
u64 par; \
11901217
asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); \

‎arch/arm64/kernel/cpu_errata.c

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -557,6 +557,13 @@ static const struct midr_range erratum_ac03_cpu_38_list[] = {
557557
};
558558
#endif
559559

560+
#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
561+
static const struct midr_range erratum_ac04_cpu_23_list[] = {
562+
MIDR_ALL_VERSIONS(MIDR_AMPERE1A),
563+
{},
564+
};
565+
#endif
566+
560567
const struct arm64_cpu_capabilities arm64_errata[] = {
561568
#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
562569
{
@@ -875,6 +882,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
875882
.capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_38,
876883
ERRATA_MIDR_RANGE_LIST(erratum_ac03_cpu_38_list),
877884
},
885+
#endif
886+
#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
887+
{
888+
.desc = "AmpereOne erratum AC04_CPU_23",
889+
.capability = ARM64_WORKAROUND_AMPERE_AC04_CPU_23,
890+
ERRATA_MIDR_RANGE_LIST(erratum_ac04_cpu_23_list),
891+
},
878892
#endif
879893
{
880894
.desc = "Broken CNTVOFF_EL2",

‎arch/arm64/kernel/hyp-stub.S

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ SYM_CODE_START_LOCAL(__finalise_el2)
9797
2:
9898
// Engage the VHE magic!
9999
mov_q x0, HCR_HOST_VHE_FLAGS
100-
msr hcr_el2, x0
100+
msr_hcr_el2 x0
101101
isb
102102

103103
// Use the EL1 allocated stack, per-cpu offset

‎arch/arm64/kvm/at.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ static void __mmu_config_save(struct mmu_config *config)
516516

517517
static void __mmu_config_restore(struct mmu_config *config)
518518
{
519-
write_sysreg(config->hcr, hcr_el2);
519+
write_sysreg_hcr(config->hcr);
520520

521521
/*
522522
* ARM errata 1165522 and 1530923 require TGE to be 1 before
@@ -1267,7 +1267,7 @@ static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
12671267

12681268
skip_mmu_switch:
12691269
/* Clear TGE, enable S2 translation, we're rolling */
1270-
write_sysreg((config.hcr & ~HCR_TGE) | HCR_VM, hcr_el2);
1270+
write_sysreg_hcr((config.hcr & ~HCR_TGE) | HCR_VM);
12711271
isb();
12721272

12731273
switch (op) {
@@ -1350,7 +1350,7 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
13501350
if (!vcpu_el2_e2h_is_set(vcpu))
13511351
val |= HCR_NV | HCR_NV1;
13521352

1353-
write_sysreg(val, hcr_el2);
1353+
write_sysreg_hcr(val);
13541354
isb();
13551355

13561356
par = SYS_PAR_EL1_F;
@@ -1375,7 +1375,7 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
13751375
if (!fail)
13761376
par = read_sysreg_par();
13771377

1378-
write_sysreg(hcr, hcr_el2);
1378+
write_sysreg_hcr(hcr);
13791379
isb();
13801380
}
13811381

‎arch/arm64/kvm/hyp/include/hyp/switch.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr)
301301
if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
302302
hcr |= HCR_TVM;
303303

304-
write_sysreg(hcr, hcr_el2);
304+
write_sysreg_hcr(hcr);
305305

306306
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
307307
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);

‎arch/arm64/kvm/hyp/nvhe/host.S

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ SYM_FUNC_START(__hyp_do_panic)
124124
/* Ensure host stage-2 is disabled */
125125
mrs x0, hcr_el2
126126
bic x0, x0, #HCR_VM
127-
msr hcr_el2, x0
127+
msr_hcr_el2 x0
128128
isb
129129
tlbi vmalls12e1
130130
dsb nsh

0 commit comments

Comments
 (0)