From 2b8e49f0fa595f8489aef1ebfde4716aede02a89 Mon Sep 17 00:00:00 2001
From: Daniel Leung <daniel.leung@intel.com>
Date: Mon, 28 Oct 2024 13:21:22 -0700
Subject: [PATCH 01/21] x86: add exception handling for control protection
 exception

This adds exception handling of control protection exception
in fatal code.

Signed-off-by: Daniel Leung <daniel.leung@intel.com>
Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/fatal.c               | 3 +++
 arch/x86/core/ia32/fatal.c          | 1 +
 arch/x86/core/intel64/locore.S      | 4 ++--
 arch/x86/gen_idt.py                 | 2 +-
 arch/x86/include/kernel_arch_data.h | 1 +
 5 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/arch/x86/core/fatal.c b/arch/x86/core/fatal.c
index 6511f531e3df6..614d40155ebbc 100644
--- a/arch/x86/core/fatal.c
+++ b/arch/x86/core/fatal.c
@@ -343,6 +343,9 @@ static void log_exception(uintptr_t vector, uintptr_t code)
 	case IV_VIRT_EXCEPTION:
 		LOG_ERR("Virtualization exception");
 		break;
+	case IV_CTRL_PROTECTION_EXCEPTION:
+		LOG_ERR("Control protection exception (code 0x%lx)", code);
+		break;
 	case IV_SECURITY_EXCEPTION:
 		LOG_ERR("Security exception");
 		break;
diff --git a/arch/x86/core/ia32/fatal.c b/arch/x86/core/ia32/fatal.c
index 3ae8a6b67da3a..1040243d60a5d 100644
--- a/arch/x86/core/ia32/fatal.c
+++ b/arch/x86/core/ia32/fatal.c
@@ -115,6 +115,7 @@ EXC_FUNC_CODE(IV_GENERAL_PROTECTION, 0);
 EXC_FUNC_NOCODE(IV_X87_FPU_FP_ERROR, 0);
 EXC_FUNC_CODE(IV_ALIGNMENT_CHECK, 0);
 EXC_FUNC_NOCODE(IV_MACHINE_CHECK, 0);
+EXC_FUNC_CODE(IV_CTRL_PROTECTION_EXCEPTION, 0);
 #endif
 
 _EXCEPTION_CONNECT_CODE(z_x86_page_fault_handler, IV_PAGE_FAULT, 0);
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 108d9f15d378f..3ed0f5cd65d0e 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -646,7 +646,7 @@ EXCEPT      ( 4, 7); EXCEPT      ( 5, 7); EXCEPT       (6, 7); EXCEPT      ( 7,
 EXCEPT_CODE ( 8, 7); EXCEPT      ( 9, 7); EXCEPT_CODE (10, 7); EXCEPT_CODE (11, 7)
 EXCEPT_CODE (12, 7); EXCEPT_CODE (13, 7); EXCEPT_CODE (14, 7); EXCEPT      (15, 7)
 EXCEPT      (16, 7); EXCEPT_CODE (17, 7); EXCEPT      (18, 7); EXCEPT      (19, 7)
-EXCEPT      (20, 7); EXCEPT      (21, 7); EXCEPT      (22, 7); EXCEPT      (23, 7)
+EXCEPT      (20, 7); EXCEPT_CODE (21, 7); EXCEPT      (22, 7); EXCEPT      (23, 7)
 EXCEPT      (24, 7); EXCEPT      (25, 7); EXCEPT      (26, 7); EXCEPT      (27, 7)
 EXCEPT      (28, 7); EXCEPT      (29, 7); EXCEPT      (30, 7); EXCEPT      (31, 7)
 
@@ -660,7 +660,7 @@ EXCEPT      ( 4); EXCEPT      ( 5); EXCEPT      ( 6); EXCEPT      ( 7)
 EXCEPT_CODE ( 8); EXCEPT      ( 9); EXCEPT_CODE (10); EXCEPT_CODE (11)
 EXCEPT_CODE (12); EXCEPT_CODE (13); EXCEPT_CODE (14); EXCEPT      (15)
 EXCEPT      (16); EXCEPT_CODE (17); EXCEPT      (18); EXCEPT      (19)
-EXCEPT      (20); EXCEPT      (21); EXCEPT      (22); EXCEPT      (23)
+EXCEPT      (20); EXCEPT_CODE (21); EXCEPT      (22); EXCEPT      (23)
 EXCEPT      (24); EXCEPT      (25); EXCEPT      (26); EXCEPT      (27)
 EXCEPT      (28); EXCEPT      (29); EXCEPT      (30); EXCEPT      (31)
 
diff --git a/arch/x86/gen_idt.py b/arch/x86/gen_idt.py
index 3ef957c952de6..7de21ea84f9cb 100755
--- a/arch/x86/gen_idt.py
+++ b/arch/x86/gen_idt.py
@@ -44,7 +44,7 @@
 KERNEL_CODE_SEG = 0x08
 
 # These exception vectors push an error code onto the stack.
-ERR_CODE_VECTORS = [8, 10, 11, 12, 13, 14, 17]
+ERR_CODE_VECTORS = [8, 10, 11, 12, 13, 14, 17, 21]
 
 
 def debug(text):
diff --git a/arch/x86/include/kernel_arch_data.h b/arch/x86/include/kernel_arch_data.h
index 7b32e125d28a9..cf9b3eb0169f7 100644
--- a/arch/x86/include/kernel_arch_data.h
+++ b/arch/x86/include/kernel_arch_data.h
@@ -32,6 +32,7 @@
 #define IV_MACHINE_CHECK 18
 #define IV_SIMD_FP 19
 #define IV_VIRT_EXCEPTION 20
+#define IV_CTRL_PROTECTION_EXCEPTION 21
 #define IV_SECURITY_EXCEPTION 30
 
 #define IV_IRQS 32		/* start of vectors available for IRQs */

From 5026b07764d8ffd6a05c6a906c22e724d311a4ee Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 12:19:43 -0700
Subject: [PATCH 02/21] arch/x86: Indirect Branch Tracking support

Indirect Branch Tracking (IBT) is one of the capabilities provided by
Intel Control-flow Enforcement Technology (CET), aimed at defending
against Jump/Call Oriented Programming.

This patch enables it for x86 (32-bit, 64-bit support coming in future
patches):

    - Add relevant Kconfigs (everything is behind X86_CET);
    - Code to enable it;
    - Enable compiler flags to enable it;
    - Add `endbr32` instructions to asm code, where needed.

Points in the code where an indirect branch is expected to land need
special instructions that tell the CPU they are valid indirect branch
targets. Those are added by the compiler, so toolchain support is
necessary. Note that any code added to the final ELF also need those
markers, such as libc or libgcc.

Finally, tests added to ensure IBT behaves sanely.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/CMakeLists.txt                    |  4 ++
 arch/x86/Kconfig                           | 15 +++++++
 arch/x86/core/CMakeLists.txt               |  2 +
 arch/x86/core/cet.c                        | 33 ++++++++++++++
 arch/x86/core/ia32/crt0.S                  |  7 +++
 arch/x86/core/ia32/excstub.S               |  1 +
 arch/x86/core/ia32/intstub.S               |  7 +++
 arch/x86/include/cet.h                     | 25 +++++++++++
 arch/x86/include/ia32/exception.h          |  1 +
 include/zephyr/arch/x86/ia32/arch.h        |  3 +-
 include/zephyr/arch/x86/msr.h              |  9 ++++
 tests/arch/x86/cet/CMakeLists.txt          |  9 ++++
 tests/arch/x86/cet/prj.conf                |  3 ++
 tests/arch/x86/cet/src/asm.S               | 26 +++++++++++
 tests/arch/x86/cet/src/main.c              | 51 ++++++++++++++++++++++
 tests/arch/x86/cet/testcase.yaml           | 12 +++++
 tests/arch/x86/static_idt/src/test_stubs.S |  1 +
 17 files changed, 208 insertions(+), 1 deletion(-)
 create mode 100644 arch/x86/core/cet.c
 create mode 100644 arch/x86/include/cet.h
 create mode 100644 tests/arch/x86/cet/CMakeLists.txt
 create mode 100644 tests/arch/x86/cet/prj.conf
 create mode 100644 tests/arch/x86/cet/src/asm.S
 create mode 100644 tests/arch/x86/cet/src/main.c
 create mode 100644 tests/arch/x86/cet/testcase.yaml

diff --git a/arch/x86/CMakeLists.txt b/arch/x86/CMakeLists.txt
index 9f46f9d149e1b..3bfb743f2421f 100644
--- a/arch/x86/CMakeLists.txt
+++ b/arch/x86/CMakeLists.txt
@@ -72,3 +72,7 @@ if(CONFIG_ARCH_HAS_TIMING_FUNCTIONS AND
     NOT CONFIG_BOARD_HAS_TIMING_FUNCTIONS)
 zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c)
 endif()
+
+if(CONFIG_X86_CET)
+  zephyr_compile_options(-fcf-protection=full)
+endif()
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 27fa6e40dcc27..121ffe41c9908 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -111,6 +111,9 @@ config X86_CPU_HAS_SSE42
 config X86_CPU_HAS_SSE4A
 	bool
 
+config X86_CPU_HAS_CET
+	bool
+
 if FPU || X86_64
 
 config X86_MMX
@@ -492,6 +495,18 @@ config PRIVILEGED_STACK_SIZE
 	# Must be multiple of CONFIG_MMU_PAGE_SIZE
 	default 4096 if X86_MMU
 
+config X86_CET
+	bool "Control-flow Enforcement Technology"
+	depends on X86_CPU_HAS_CET
+	help
+	  This option enables Control-flow Enforcement Technology (CET) support.
+
+config X86_CET_IBT
+	bool "Indirect Branch Tracking"
+	depends on X86_CET && PICOLIBC_USE_MODULE
+	help
+	  This option enables Indirect Branch Tracking (IBT) support.
+
 source "arch/x86/core/Kconfig.ia32"
 source "arch/x86/core/Kconfig.intel64"
 
diff --git a/arch/x86/core/CMakeLists.txt b/arch/x86/core/CMakeLists.txt
index c34c6909b0172..80200f4015b57 100644
--- a/arch/x86/core/CMakeLists.txt
+++ b/arch/x86/core/CMakeLists.txt
@@ -29,6 +29,8 @@ zephyr_library_sources_ifdef(CONFIG_THREAD_LOCAL_STORAGE tls.c)
 
 zephyr_library_sources_ifdef(CONFIG_LLEXT elf.c)
 
+zephyr_library_sources_ifdef(CONFIG_X86_CET cet.c)
+
 if(CONFIG_X86_64)
   include(intel64.cmake)
 else()
diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
new file mode 100644
index 0000000000000..5e27e94b33af7
--- /dev/null
+++ b/arch/x86/core/cet.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @file
+ * @brief Indirect Branch Tracking
+ *
+ * Indirect Branch Tracking (IBT) setup routines.
+ */
+
+#include <zephyr/kernel.h>
+#include <zephyr/arch/x86/msr.h>
+
+void z_x86_cet_enable(void)
+{
+	__asm volatile (
+			"movl %cr4, %eax\n\t"
+			"orl $0x800000, %eax\n\t"
+			"movl %eax, %cr4\n\t"
+			);
+}
+
+#ifdef CONFIG_X86_CET_IBT
+void z_x86_ibt_enable(void)
+{
+	uint64_t msr = z_x86_msr_read(X86_S_CET_MSR);
+
+	msr |= X86_S_CET_MSR_ENDBR | X86_S_CET_MSR_NO_TRACK;
+	z_x86_msr_write(X86_S_CET_MSR, msr);
+}
+#endif
diff --git a/arch/x86/core/ia32/crt0.S b/arch/x86/core/ia32/crt0.S
index 0c7ea821280b9..68b1d643eef94 100644
--- a/arch/x86/core/ia32/crt0.S
+++ b/arch/x86/core/ia32/crt0.S
@@ -278,6 +278,13 @@ __csSet:
 	call	z_bss_zero
 #endif
 
+#ifdef CONFIG_X86_CET
+	call z_x86_cet_enable
+#ifdef CONFIG_X86_CET_IBT
+	call z_x86_ibt_enable
+#endif
+#endif
+
 	/* load 32-bit operand size IDT */
 	lidt	z_x86_idt
 
diff --git a/arch/x86/core/ia32/excstub.S b/arch/x86/core/ia32/excstub.S
index 6c0a13a37cde3..9ca3dacc053f4 100644
--- a/arch/x86/core/ia32/excstub.S
+++ b/arch/x86/core/ia32/excstub.S
@@ -233,6 +233,7 @@ nestedException:
 	KPTI_IRET
 
 SECTION_FUNC(PINNED_TEXT, _kernel_oops_handler)
+	endbr32
 	push $0 /* dummy error code */
 	push $z_x86_do_kernel_oops
 	jmp _exception_enter
diff --git a/arch/x86/core/ia32/intstub.S b/arch/x86/core/ia32/intstub.S
index dd454670dd886..57c45a15f4174 100644
--- a/arch/x86/core/ia32/intstub.S
+++ b/arch/x86/core/ia32/intstub.S
@@ -357,6 +357,7 @@ handle_idle:
  */
 SECTION_FUNC(PINNED_TEXT, z_SpuriousIntNoErrCodeHandler)
 
+	endbr32
 	pushl	$0			/* push dummy err code onto stk */
 
 	/* fall through to z_SpuriousIntHandler */
@@ -364,6 +365,7 @@ SECTION_FUNC(PINNED_TEXT, z_SpuriousIntNoErrCodeHandler)
 
 SECTION_FUNC(PINNED_TEXT, z_SpuriousIntHandler)
 
+	endbr32
 	cld				/* Clear direction flag */
 
 	/* Create the ESF */
@@ -391,6 +393,7 @@ SECTION_FUNC(PINNED_TEXT, z_SpuriousIntHandler)
 
 #if CONFIG_IRQ_OFFLOAD
 SECTION_FUNC(PINNED_TEXT, _irq_sw_handler)
+	endbr32
 	push $0
 	push $z_irq_do_offload
 	jmp _interrupt_enter
@@ -429,6 +432,10 @@ stub_num = 0
 	.rept Z_DYN_STUB_PER_BLOCK
 		.if stub_num < CONFIG_X86_DYNAMIC_IRQ_STUBS
 			INT_STUB_NUM stub_num
+			/*
+			 * 4-byte endbr32
+			 */
+			endbr32
 			/*
 			 * 2-byte push imm8.
 			 */
diff --git a/arch/x86/include/cet.h b/arch/x86/include/cet.h
new file mode 100644
index 0000000000000..c832f4816f807
--- /dev/null
+++ b/arch/x86/include/cet.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+#ifndef ZEPHYR_ARCH_X86_INCLUDE_CET_H
+#define ZEPHYR_ARCH_X86_INCLUDE_CET_H
+
+#ifndef _ASMLANGUAGE
+
+/* Enables Control-flow Enforcement Technology (CET). Currently only
+ * supported in 32-bit mode.
+ */
+void z_x86_cet_enable(void);
+
+#ifdef CONFIG_X86_CET_IBT
+/* Enables Indirect Branch Tracking (IBT).
+ */
+void z_x86_ibt_enable(void);
+#endif
+
+#endif /* _ASMLANGUAGE */
+#endif /* ZEPHYR_ARCH_X86_INCLUDE_CET_H */
diff --git a/arch/x86/include/ia32/exception.h b/arch/x86/include/ia32/exception.h
index 1b0ce9ee3b52d..86de44f76c0bb 100644
--- a/arch/x86/include/ia32/exception.h
+++ b/arch/x86/include/ia32/exception.h
@@ -49,6 +49,7 @@
 	".global " STRINGIFY(_EXCEPTION_STUB_NAME(handler, vector)) "\n\t" \
 	STRINGIFY(_EXCEPTION_STUB_NAME(handler, vector)) ":\n\t" \
 	"1:\n\t" \
+	"endbr32\n\t" \
 	codepush \
 	"push $" STRINGIFY(handler) "\n\t" \
 	"jmp _exception_enter\n\t" \
diff --git a/include/zephyr/arch/x86/ia32/arch.h b/include/zephyr/arch/x86/ia32/arch.h
index 6b90fd2737ab4..7a6e37184027c 100644
--- a/include/zephyr/arch/x86/ia32/arch.h
+++ b/include/zephyr/arch/x86/ia32/arch.h
@@ -59,7 +59,7 @@
  */
 #define MK_ISR_NAME(x) __isr__##x
 
-#define Z_DYN_STUB_SIZE			4
+#define Z_DYN_STUB_SIZE			8
 #define Z_DYN_STUB_OFFSET		0
 #define Z_DYN_STUB_LONG_JMP_EXTRA_SIZE	3
 #define Z_DYN_STUB_PER_BLOCK		32
@@ -210,6 +210,7 @@ typedef struct s_isrList {
 		".pushsection " IRQSTUBS_TEXT_SECTION "\n\t" \
 		".global %c[isr]_irq%c[irq]_stub\n\t" \
 		"%c[isr]_irq%c[irq]_stub:\n\t" \
+		"endbr32\n\t" \
 		"pushl %[isr_param]\n\t" \
 		"pushl %[isr]\n\t" \
 		"jmp _interrupt_enter\n\t" \
diff --git a/include/zephyr/arch/x86/msr.h b/include/zephyr/arch/x86/msr.h
index 0ccd3806bbfb5..beddcf9408009 100644
--- a/include/zephyr/arch/x86/msr.h
+++ b/include/zephyr/arch/x86/msr.h
@@ -6,6 +6,8 @@
 #ifndef ZEPHYR_INCLUDE_ARCH_X86_MSR_H_
 #define ZEPHYR_INCLUDE_ARCH_X86_MSR_H_
 
+#include <zephyr/toolchain.h>
+
 /*
  * Model specific registers (MSR).  Access with z_x86_msr_read/write().
  */
@@ -24,6 +26,13 @@
 
 #define X86_X2APIC_BASE_MSR		0x00000800 /* .. thru 0x00000BFF */
 
+#define X86_U_CET_MSR			0x000006A0
+
+#define X86_S_CET_MSR			0x000006A2
+#define X86_S_CET_MSR_SHSTK		BIT(0)
+#define X86_S_CET_MSR_ENDBR		BIT(2)
+#define X86_S_CET_MSR_NO_TRACK		BIT(4)
+
 #define X86_EFER_MSR			0xC0000080U
 #define X86_EFER_MSR_SCE		BIT(0)
 #define X86_EFER_MSR_LME		BIT(8)
diff --git a/tests/arch/x86/cet/CMakeLists.txt b/tests/arch/x86/cet/CMakeLists.txt
new file mode 100644
index 0000000000000..6223e8cf7c137
--- /dev/null
+++ b/tests/arch/x86/cet/CMakeLists.txt
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(cet)
+
+enable_language(C ASM)
+
+target_sources(app PRIVATE src/main.c src/asm.S)
diff --git a/tests/arch/x86/cet/prj.conf b/tests/arch/x86/cet/prj.conf
new file mode 100644
index 0000000000000..17401e756ae6a
--- /dev/null
+++ b/tests/arch/x86/cet/prj.conf
@@ -0,0 +1,3 @@
+CONFIG_ZTEST=y
+CONFIG_X86_CET=y
+CONFIG_DEBUG_COREDUMP=y
diff --git a/tests/arch/x86/cet/src/asm.S b/tests/arch/x86/cet/src/asm.S
new file mode 100644
index 0000000000000..193e51edeb18c
--- /dev/null
+++ b/tests/arch/x86/cet/src/asm.S
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <zephyr/arch/x86/ia32/asm.h>
+
+GTEXT(should_work)
+GTEXT(should_not_work)
+
+should_work:
+    endbr32
+    push %ebp
+    mov %esp, %ebp
+    mov 0x8(%ebp), %eax
+    inc %eax
+    pop %ebp
+    ret
+
+should_not_work:
+    push %ebp
+    mov %esp, %ebp
+    mov 0x8(%ebp), %eax
+    inc %eax
+    pop %ebp
+    ret
diff --git a/tests/arch/x86/cet/src/main.c b/tests/arch/x86/cet/src/main.c
new file mode 100644
index 0000000000000..094d99bf379c3
--- /dev/null
+++ b/tests/arch/x86/cet/src/main.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+#include <zephyr/arch/exception.h>
+
+#define IV_CTRL_PROTECTION_EXCEPTION 21
+
+#define CTRL_PROTECTION_ERRORCODE_ENDBRANCH 3
+
+extern int should_work(int a);
+extern int should_not_work(int a);
+
+static bool expect_fault;
+static int expect_code;
+
+void k_sys_fatal_error_handler(unsigned int reason, const struct arch_esf *pEsf)
+{
+	if (expect_fault) {
+		zassert_equal(z_x86_exception_vector, IV_CTRL_PROTECTION_EXCEPTION,
+					  "unexpected exception");
+		zassert_equal(pEsf->errorCode, expect_code, "unexpected error code");
+		printk("fatal error expected as part of test case\n");
+		expect_fault = false;
+	} else {
+		printk("fatal error was unexpected, aborting\n");
+		TC_END_REPORT(TC_FAIL);
+		k_fatal_halt(reason);
+	}
+}
+
+/* Round trip to trick optimisations and ensure the calls are indirect */
+int do_call(int (*func)(int), int a)
+{
+	return func(a);
+}
+
+ZTEST(cet, test_ibt)
+{
+	zassert_equal(do_call(should_work, 1), 2, "should_work failed");
+
+	expect_fault = true;
+	expect_code = CTRL_PROTECTION_ERRORCODE_ENDBRANCH;
+	do_call(should_not_work, 1);
+	zassert_unreachable("should_not_work did not fault");
+}
+
+ZTEST_SUITE(cet, NULL, NULL, NULL, NULL, NULL);
diff --git a/tests/arch/x86/cet/testcase.yaml b/tests/arch/x86/cet/testcase.yaml
new file mode 100644
index 0000000000000..89c2af035eddc
--- /dev/null
+++ b/tests/arch/x86/cet/testcase.yaml
@@ -0,0 +1,12 @@
+common:
+  ignore_faults: true
+  tags:
+    - kernel
+    - cet
+  filter: CONFIG_X86_CPU_HAS_CET
+
+tests:
+  arch.x86.cet.kernel.ibt:
+    arch_allow: x86
+    extra_configs:
+      - CONFIG_X86_CET_IBT=y
diff --git a/tests/arch/x86/static_idt/src/test_stubs.S b/tests/arch/x86/static_idt/src/test_stubs.S
index cbab47b7f5767..1656bd733f6a7 100644
--- a/tests/arch/x86/static_idt/src/test_stubs.S
+++ b/tests/arch/x86/static_idt/src/test_stubs.S
@@ -26,6 +26,7 @@ testing.
 
 	GTEXT(int_stub)
 SECTION_FUNC(TEXT, int_stub)
+        endbr32
         pushl   $0
         pushl   $isr_handler
         jmp     _interrupt_enter

From 558ecbf80fb48373df8a147798cd53f98dd87ce3 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Feb 2025 15:43:17 -0800
Subject: [PATCH 03/21] arch/x86: Extend IBT support to x86_64

Add code to enable it and sprinkle `endbr64` on asm code, where needed.
Namely, IRQs and excepts entrypoints.

Finally, tests added to ensure IBT behaves sanely.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/cet.c              |  8 ++++++++
 arch/x86/core/intel64/locore.S   | 10 +++++-----
 tests/arch/x86/cet/prj.conf      |  1 -
 tests/arch/x86/cet/src/asm.S     | 13 +++++++++++++
 tests/arch/x86/cet/src/main.c    |  6 ++++++
 tests/arch/x86/cet/testcase.yaml |  9 ++++++++-
 6 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 5e27e94b33af7..778db1844516c 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -15,11 +15,19 @@
 
 void z_x86_cet_enable(void)
 {
+#ifdef CONFIG_X86_64
+	__asm volatile (
+			"movq %cr4, %rax\n\t"
+			"orq $0x800000, %rax\n\t"
+			"movq %rax, %cr4\n\t"
+			);
+#else
 	__asm volatile (
 			"movl %cr4, %eax\n\t"
 			"orl $0x800000, %eax\n\t"
 			"movl %eax, %cr4\n\t"
 			);
+#endif
 }
 
 #ifdef CONFIG_X86_CET_IBT
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 3ed0f5cd65d0e..6c29267e4e7dc 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -498,17 +498,17 @@ __resume:
 
 #ifdef CONFIG_X86_KPTI
 #define EXCEPT_CODE(nr, ist)						\
-	vector_ ## nr: pushq %gs:__x86_tss64_t_ist ## ist ## _OFFSET;	\
+	vector_ ## nr: endbr64; pushq %gs:__x86_tss64_t_ist ## ist ## _OFFSET;	\
 	pushq $nr;							\
 	jmp except
 #define EXCEPT(nr, ist)							\
-	vector_ ## nr: pushq $0;					\
+	vector_ ## nr: endbr64; pushq $0;					\
 	pushq %gs:__x86_tss64_t_ist ## ist ## _OFFSET;			\
 	pushq $nr; \
 	jmp except
 #else
-#define EXCEPT_CODE(nr) vector_ ## nr: pushq $nr; jmp except
-#define EXCEPT(nr)	vector_ ## nr: pushq $0; pushq $nr; jmp except
+#define EXCEPT_CODE(nr) vector_ ## nr: endbr64; pushq $nr; jmp except
+#define EXCEPT(nr)	vector_ ## nr: endbr64; pushq $0; pushq $nr; jmp except
 #endif
 
 /*
@@ -849,7 +849,7 @@ irq_exit_nested:
 	popq %rax
 	iretq
 
-#define IRQ(nr) vector_ ## nr: pushq $(nr - IV_IRQS); jmp irq
+#define IRQ(nr) vector_ ## nr: endbr64; pushq $(nr - IV_IRQS); jmp irq
 
 IRQ( 33); IRQ( 34); IRQ( 35); IRQ( 36); IRQ( 37); IRQ( 38); IRQ( 39)
 IRQ( 40); IRQ( 41); IRQ( 42); IRQ( 43); IRQ( 44); IRQ( 45); IRQ( 46); IRQ( 47)
diff --git a/tests/arch/x86/cet/prj.conf b/tests/arch/x86/cet/prj.conf
index 17401e756ae6a..54be14f4093c9 100644
--- a/tests/arch/x86/cet/prj.conf
+++ b/tests/arch/x86/cet/prj.conf
@@ -1,3 +1,2 @@
 CONFIG_ZTEST=y
 CONFIG_X86_CET=y
-CONFIG_DEBUG_COREDUMP=y
diff --git a/tests/arch/x86/cet/src/asm.S b/tests/arch/x86/cet/src/asm.S
index 193e51edeb18c..4aaec318fdbb3 100644
--- a/tests/arch/x86/cet/src/asm.S
+++ b/tests/arch/x86/cet/src/asm.S
@@ -8,6 +8,18 @@
 GTEXT(should_work)
 GTEXT(should_not_work)
 
+#if defined(CONFIG_X86_64)
+should_work:
+    endbr64
+    mov %rdi, %rax
+    inc %rax
+    ret
+
+should_not_work:
+    mov %rdi, %rax
+    inc %rax
+    ret
+#else
 should_work:
     endbr32
     push %ebp
@@ -24,3 +36,4 @@ should_not_work:
     inc %eax
     pop %ebp
     ret
+#endif
diff --git a/tests/arch/x86/cet/src/main.c b/tests/arch/x86/cet/src/main.c
index 094d99bf379c3..4c848528d57aa 100644
--- a/tests/arch/x86/cet/src/main.c
+++ b/tests/arch/x86/cet/src/main.c
@@ -20,9 +20,15 @@ static int expect_code;
 void k_sys_fatal_error_handler(unsigned int reason, const struct arch_esf *pEsf)
 {
 	if (expect_fault) {
+#ifdef CONFIG_X86_64
+		zassert_equal(pEsf->vector, IV_CTRL_PROTECTION_EXCEPTION,
+					  "unexpected exception");
+		zassert_equal(pEsf->code, expect_code, "unexpected error code");
+#else
 		zassert_equal(z_x86_exception_vector, IV_CTRL_PROTECTION_EXCEPTION,
 					  "unexpected exception");
 		zassert_equal(pEsf->errorCode, expect_code, "unexpected error code");
+#endif
 		printk("fatal error expected as part of test case\n");
 		expect_fault = false;
 	} else {
diff --git a/tests/arch/x86/cet/testcase.yaml b/tests/arch/x86/cet/testcase.yaml
index 89c2af035eddc..ce4d5630b92b9 100644
--- a/tests/arch/x86/cet/testcase.yaml
+++ b/tests/arch/x86/cet/testcase.yaml
@@ -6,7 +6,14 @@ common:
   filter: CONFIG_X86_CPU_HAS_CET
 
 tests:
-  arch.x86.cet.kernel.ibt:
+  arch.x86.cet.kernel.ibt32:
     arch_allow: x86
+    filter: not CONFIG_X86_64
+    extra_configs:
+      - CONFIG_X86_CET_IBT=y
+      - CONFIG_DEBUG_COREDUMP=y
+  arch.x86.cet.kernel.ibt64:
+    arch_allow: x86
+    filter: CONFIG_X86_64
     extra_configs:
       - CONFIG_X86_CET_IBT=y

From 26e9d46be731230f0c5a791cb89d5f582587b1d4 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 4 Mar 2025 15:35:44 -0800
Subject: [PATCH 04/21] arch/x86: Shadow Stack support

Shadow Stack is one of the capabilities provided by Intel Control-flow
Enforcement Technology (CET), aimed at defending against Return Oriented
Programming.

This patch enables it for x86_64 (32-bit support coming in future
patches):

    - Add relevant Kconfigs;
    - Shadow stacks should live in specially defined memory pages, so
      gen_mmu.py was updated to allow that;
    - A new macro, Z_X86_SHADOW_STACK_DEFINE, added to define the area
      for a shadow stack;
    - A new function, z_x86_thread_attach_shadow_stack(), added to
      attach a shadow stack to a never started thread;
    - locore.S changed to enable/disable shadow stack when a thread
      using it comes in/out of execution.

As not all threads are currently shadow stack capable, threads that do
not use it will still run with shadow stack disabled. Ideally, at some
point in the future, all threads would use the shadow stack, so no need
to disable it at all.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/Kconfig                              | 14 ++++
 arch/x86/core/cet.c                           | 31 +++++++++
 arch/x86/core/fatal.c                         |  8 +++
 arch/x86/core/intel64/cpu.c                   |  8 +++
 arch/x86/core/intel64/locore.S                | 64 ++++++++++++++++++-
 arch/x86/core/offsets/intel64_offsets.c       |  5 ++
 arch/x86/gen_mmu.py                           | 11 +++-
 arch/x86/include/intel64/offsets_short_arch.h | 11 ++++
 include/zephyr/arch/x86/cet.h                 | 44 +++++++++++++
 include/zephyr/arch/x86/intel64/linker.ld     | 16 +++++
 include/zephyr/arch/x86/intel64/thread.h      |  6 ++
 include/zephyr/arch/x86/msr.h                 |  4 ++
 12 files changed, 219 insertions(+), 3 deletions(-)
 create mode 100644 include/zephyr/arch/x86/cet.h

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 121ffe41c9908..d7caff60b0e19 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -507,6 +507,20 @@ config X86_CET_IBT
 	help
 	  This option enables Indirect Branch Tracking (IBT) support.
 
+config X86_CET_SHADOW_STACK
+	bool "Shadow Stack"
+	depends on X86_CET
+	depends on SRAM_REGION_PERMISSIONS
+	help
+	  This option enables Shadow Stack support.
+
+config X86_CET_SHADOW_STACK_ALIGNMENT
+	int "Shadow Stack alignment"
+	default 4096
+	depends on X86_CET_SHADOW_STACK
+	help
+	  This option sets the alignment of the Shadow Stack.
+
 source "arch/x86/core/Kconfig.ia32"
 source "arch/x86/core/Kconfig.intel64"
 
diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 778db1844516c..9c54e7a998503 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -12,6 +12,37 @@
 
 #include <zephyr/kernel.h>
 #include <zephyr/arch/x86/msr.h>
+#include <zephyr/arch/x86/cet.h>
+#include <zephyr/logging/log_ctrl.h>
+#include <zephyr/logging/log.h>
+
+LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+int z_x86_thread_attach_shadow_stack(k_tid_t thread,
+				     z_x86_shadow_stack_t *stack,
+				     size_t stack_size)
+{
+	/* Can't attach to NULL */
+	if (stack == NULL) {
+		LOG_ERR("Can't set NULL shadow stack for thread %p\n", thread);
+		return -EINVAL;
+	}
+
+	/* Or if the thread already has a shadow stack. */
+	if (thread->arch.shstk_addr != NULL) {
+		LOG_ERR("Shadow stack already set up for thread %p\n", thread);
+		return -EINVAL;
+	}
+
+	thread->arch.shstk_addr = stack + (stack_size -
+					   5 * sizeof(*stack)) / sizeof(*stack);
+	thread->arch.shstk_size = stack_size;
+	thread->arch.shstk_base = stack;
+
+	return 0;
+}
+#endif
 
 void z_x86_cet_enable(void)
 {
diff --git a/arch/x86/core/fatal.c b/arch/x86/core/fatal.c
index 614d40155ebbc..6955623045b6d 100644
--- a/arch/x86/core/fatal.c
+++ b/arch/x86/core/fatal.c
@@ -265,6 +265,14 @@ static void dump_regs(const struct arch_esf *esf)
 		esf->rsp, esf->rflags, esf->cs & 0xFFFFU, get_cr3(esf));
 
 	LOG_ERR("RIP: 0x%016lx", esf->rip);
+#ifdef CONFIG_HW_SHADOW_STACK
+	{
+	uintptr_t ssp;
+
+		__asm__ volatile("rdsspq %0" : "=r"(ssp));
+		LOG_ERR("SSP: 0x%016lx", ssp);
+	}
+#endif /* CONFIG_HW_SHADOW_STACK */
 }
 #else /* 32-bit */
 __pinned_func
diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c
index 31dbf060d6c9c..54679cdc1d5a3 100644
--- a/arch/x86/core/intel64/cpu.c
+++ b/arch/x86/core/intel64/cpu.c
@@ -13,6 +13,7 @@
 #include <zephyr/arch/x86/multiboot.h>
 #include <x86_mmu.h>
 #include <zephyr/drivers/interrupt_controller/loapic.h>
+#include <cet.h>
 #ifdef CONFIG_ACPI
 #include <zephyr/arch/x86/cpuid.h>
 #include <zephyr/acpi/acpi.h>
@@ -122,6 +123,13 @@ FUNC_NORETURN void z_x86_cpu_init(struct x86_cpuboot *cpuboot)
 	z_x86_msr_write(X86_FMASK_MSR, EFLAGS_SYSCALL);
 #endif
 
+#ifdef CONFIG_X86_CET
+	z_x86_cet_enable();
+#ifdef CONFIG_X86_CET_IBT
+	z_x86_ibt_enable();
+#endif /* CONFIG_X86_CET_IBT */
+#endif /* CONFIG_X86_CET */
+
 	/* Enter kernel, never return */
 	cpuboot->ready++;
 	cpuboot->fn(cpuboot->arg);
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 6c29267e4e7dc..3aa64e3415239 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -373,6 +373,46 @@ z_x86_switch:
 	movq $X86_KERNEL_CS, _thread_offset_to_cs(%rsi)
 	movq $X86_KERNEL_DS, _thread_offset_to_ss(%rsi)
 #endif
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
+	jz __sh_stk_out
+
+	/* Create the shadow stack restore token by hand, as using
+	 * saveprevssp is cumbersome when some threads don't use shadow
+	 * stack. Note that as we return using iretq, we also need to add
+	 * current CS and LIP (return pointer) to the shadow stack. It should
+	 * currently be pointing to return pointer.
+	 */
+	rdsspq %rcx
+	movq (%rcx), %r10 /* LIP */
+
+	/* Save CS, overwriting current top of stack */
+	movq $X86_KERNEL_CS, %r9
+	wrssq %r9, (%rcx)
+
+	/* Save LIP */
+	wrssq %r10, -8(%rcx)
+
+	/* Save SSP */
+	movq %rcx, %r11
+	addq $8, %r11 /* SSP has to point to before previous entry. */
+	wrssq %r11, -16(%rcx)
+
+	/* Create and "push" new token */
+	movq %rcx, %r11
+	subq $16, %r11
+	orq $1, %r11
+	wrssq %r11, -24(%rcx)
+
+	/* Finally, save SSP on thread struct */
+	subq $24, %rcx
+	movq %rcx, _thread_offset_to_shstk_addr(%rsi)
+	jmp __sh_stk_out
+
+__sh_stk_out:
+#endif
+
 	/* Store the handle (i.e. our thread struct address) into the
 	 * switch handle field, this is a synchronization signal that
 	 * must occur after the last data from the old context is
@@ -407,6 +447,29 @@ __resume:
 	shrq $32, %rdx
 	wrmsr
 #endif
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	movq _thread_offset_to_shstk_addr(%rdi), %r10
+	testq %r10, %r10
+	jz __sh_stk_disable
+
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	orl $X86_S_CET_MSR_SHSTK_EN, %eax
+	wrmsr
+	rstorssp (%r10)
+	movq $1, %r10
+	incsspq %r10 /* Ignore previous ssp token */
+	jmp __sh_stk_done
+__sh_stk_disable:
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	andl $~X86_S_CET_MSR_SHSTK_EN, %eax
+	andl $~X86_S_CET_MSR_WR_SHSTK, %eax
+	wrmsr
+__sh_stk_done:
+#endif
+
 #if (!defined(CONFIG_X86_KPTI) && defined(CONFIG_USERSPACE)) \
 		|| defined(CONFIG_INSTRUMENT_THREAD_SWITCHING)
 	pushq %rdi	/* Caller-saved, stash it */
@@ -449,7 +512,6 @@ __resume:
 	movq $0xB9, _thread_offset_to_rip(%rdi)
 #endif
 
-
 	movq _thread_offset_to_rbx(%rdi), %rbx
 	movq _thread_offset_to_rbp(%rdi), %rbp
 	movq _thread_offset_to_r12(%rdi), %r12
diff --git a/arch/x86/core/offsets/intel64_offsets.c b/arch/x86/core/offsets/intel64_offsets.c
index 1ce29aa3b32f7..60c772b76aa01 100644
--- a/arch/x86/core/offsets/intel64_offsets.c
+++ b/arch/x86/core/offsets/intel64_offsets.c
@@ -34,6 +34,11 @@ GEN_OFFSET_SYM(_thread_arch_t, psp);
 GEN_OFFSET_SYM(_thread_arch_t, ptables);
 #endif
 #endif /* CONFIG_USERSPACE */
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+GEN_OFFSET_SYM(_thread_arch_t, shstk_addr);
+GEN_OFFSET_SYM(_thread_arch_t, shstk_base);
+GEN_OFFSET_SYM(_thread_arch_t, shstk_size);
+#endif
 
 GEN_OFFSET_SYM(x86_tss64_t, ist1);
 GEN_OFFSET_SYM(x86_tss64_t, ist2);
diff --git a/arch/x86/gen_mmu.py b/arch/x86/gen_mmu.py
index e09e3208e7965..8bfe2501e5f72 100755
--- a/arch/x86/gen_mmu.py
+++ b/arch/x86/gen_mmu.py
@@ -94,6 +94,7 @@ def bit(pos):
 FLAG_RW = bit(1)
 FLAG_US = bit(2)
 FLAG_CD = bit(4)
+FLAG_D = bit(6)
 FLAG_SZ = bit(7)
 FLAG_G = bit(8)
 FLAG_XD = bit(63)
@@ -163,6 +164,9 @@ def dump_flags(flags):
     if flags & FLAG_CD:
         ret += "CD "
 
+    if flags & FLAG_D:
+        ret += "D "
+
     return ret.strip()
 
 
@@ -320,7 +324,7 @@ class Pt(MMUTable):
     addr_mask = 0xFFFFF000
     type_code = 'I'
     num_entries = 1024
-    supported_flags = (FLAG_P | FLAG_RW | FLAG_US | FLAG_G | FLAG_CD |
+    supported_flags = (FLAG_P | FLAG_RW | FLAG_US | FLAG_G | FLAG_CD | FLAG_D |
                        FLAG_IGNORED0 | FLAG_IGNORED1)
 
 class PtXd(Pt):
@@ -329,7 +333,7 @@ class PtXd(Pt):
     type_code = 'Q'
     num_entries = 512
     supported_flags = (FLAG_P | FLAG_RW | FLAG_US | FLAG_G | FLAG_XD | FLAG_CD |
-                       FLAG_IGNORED0 | FLAG_IGNORED1 | FLAG_IGNORED2)
+                       FLAG_D | FLAG_IGNORED0 | FLAG_IGNORED1 | FLAG_IGNORED2)
 
 
 class PtableSet():
@@ -909,6 +913,9 @@ def main():
             pt.set_region_perms("_locore", FLAG_P | flag_user)
             pt.set_region_perms("_lorodata", FLAG_P | ENTRY_XD | flag_user)
 
+        if isdef("CONFIG_X86_CET_SHADOW_STACK"):
+            pt.set_region_perms("__x86shadowstack", FLAG_P | FLAG_D | ENTRY_XD)
+
     written_size = pt.write_output(args.output)
     debug("Written %d bytes to %s" % (written_size, args.output))
 
diff --git a/arch/x86/include/intel64/offsets_short_arch.h b/arch/x86/include/intel64/offsets_short_arch.h
index 1ffabc899c204..6a8ab1c193a2c 100644
--- a/arch/x86/include/intel64/offsets_short_arch.h
+++ b/arch/x86/include/intel64/offsets_short_arch.h
@@ -71,4 +71,15 @@
 #define _thread_offset_to_cs \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_cs_OFFSET)
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+#define _thread_offset_to_shstk_addr \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_addr_OFFSET)
+
+#define _thread_offset_to_shstk_base \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_base_OFFSET)
+
+#define _thread_offset_to_shstk_size \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_size_OFFSET)
+#endif
+
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_INTEL64_OFFSETS_SHORT_ARCH_H_ */
diff --git a/include/zephyr/arch/x86/cet.h b/include/zephyr/arch/x86/cet.h
new file mode 100644
index 0000000000000..3b084d9c3e619
--- /dev/null
+++ b/include/zephyr/arch/x86/cet.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+#ifndef ZEPHYR_INCLUDE_ARCH_X86_CET_H_
+#define ZEPHYR_INCLUDE_ARCH_X86_CET_H_
+
+#include <zephyr/kernel.h>
+#include <zephyr/toolchain.h>
+#include <zephyr/sys/util.h>
+
+#ifndef _ASMLANGUAGE
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+
+extern FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry,
+					 void *p1, void *p2, void *p3);
+
+typedef long z_x86_shadow_stack_t;
+
+#define Z_X86_SHADOW_STACK_DEFINE(name, size) \
+	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
+	name[size / sizeof(z_x86_shadow_stack_t)] = \
+		{ [size / sizeof(z_x86_shadow_stack_t) - 5] = \
+		   (uintptr_t)name + size - 4 * sizeof(z_x86_shadow_stack_t) + 1, \
+		  [size / sizeof(z_x86_shadow_stack_t) - 4] = \
+		  (uintptr_t)name + size - 1 * sizeof(z_x86_shadow_stack_t), \
+		  [size / sizeof(z_x86_shadow_stack_t) - 3] = \
+		   (uintptr_t)z_thread_entry, \
+		  [size / sizeof(z_x86_shadow_stack_t) - 2] = \
+		   (uintptr_t)X86_KERNEL_CS }
+
+int z_x86_thread_attach_shadow_stack(k_tid_t thread,
+				     z_x86_shadow_stack_t *stack,
+				     size_t stack_size);
+
+#endif
+
+#endif /* _ASMLANGUAGE */
+#endif /* ZEPHYR_INCLUDE_ARCH_X86_CET_H_ */
diff --git a/include/zephyr/arch/x86/intel64/linker.ld b/include/zephyr/arch/x86/intel64/linker.ld
index d220d8ccac4d2..04fb2a15bc885 100644
--- a/include/zephyr/arch/x86/intel64/linker.ld
+++ b/include/zephyr/arch/x86/intel64/linker.ld
@@ -189,6 +189,22 @@ SECTIONS
 #include <zephyr/linker/cplusplus-ram.ld>
 #include <zephyr/arch/x86/pagetables.ld>
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+SECTION_PROLOGUE(.x86shadowstack,,)
+{
+    MMU_PAGE_ALIGN
+
+    __x86shadowstack_start = .;
+
+    *(.x86shadowstack)
+
+    MMU_PAGE_ALIGN
+
+    __x86shadowstack_end = .;
+
+} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+#endif
+
 /* Located in generated directory. This file is populated by the
  * zephyr_linker_sources() Cmake function.
  */
diff --git a/include/zephyr/arch/x86/intel64/thread.h b/include/zephyr/arch/x86/intel64/thread.h
index ae03c29f1a743..5ca80131c853f 100644
--- a/include/zephyr/arch/x86/intel64/thread.h
+++ b/include/zephyr/arch/x86/intel64/thread.h
@@ -135,6 +135,12 @@ struct _thread_arch {
 	uint64_t cs;
 #endif
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	uintptr_t *shstk_addr;
+	uintptr_t *shstk_base;
+	size_t shstk_size;
+#endif
+
 	uint64_t rax;
 	uint64_t rcx;
 	uint64_t rdx;
diff --git a/include/zephyr/arch/x86/msr.h b/include/zephyr/arch/x86/msr.h
index beddcf9408009..9426a563f8e31 100644
--- a/include/zephyr/arch/x86/msr.h
+++ b/include/zephyr/arch/x86/msr.h
@@ -30,9 +30,13 @@
 
 #define X86_S_CET_MSR			0x000006A2
 #define X86_S_CET_MSR_SHSTK		BIT(0)
+#define X86_S_CET_MSR_WR_SHSTK		BIT(1)
 #define X86_S_CET_MSR_ENDBR		BIT(2)
 #define X86_S_CET_MSR_NO_TRACK		BIT(4)
 
+#define X86_S_CET_MSR_SHSTK_EN		(X86_S_CET_MSR_SHSTK | \
+					 X86_S_CET_MSR_WR_SHSTK)
+
 #define X86_EFER_MSR			0xC0000080U
 #define X86_EFER_MSR_SCE		BIT(0)
 #define X86_EFER_MSR_LME		BIT(8)

From bf53887800058a93c9847f6307ed8930e38ccbee Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 15:22:19 -0700
Subject: [PATCH 05/21] arch/x86: Support shadow stack on IRQ

For IRQs, shadow stack support a mechanism similar to the Interrupt
Stack Table (IST) for x86_64: a table, indexed by the IST index, pointing
to a 64 byte table in memory containing the address of seven shadow stacks
to be used by the interrupt service routines.

This patch adds support to this mechanism. It is worth noting that, as
Zephyr may exit from an interrupt by going to a different thread than
the one that was interrupted, some housekeeping is done to ensure that
the necessary shadow stack tokens are on the right shadow stack before
return from the interrupt.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/Kconfig                            |  8 +++
 arch/x86/core/intel64/cpu.c                 | 12 +++++
 arch/x86/core/intel64/locore.S              | 59 +++++++++++++++++++++
 arch/x86/include/cet.h                      |  7 +++
 arch/x86/include/intel64/kernel_arch_data.h | 38 +++++++++++++
 include/zephyr/arch/x86/msr.h               |  2 +
 6 files changed, 126 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d7caff60b0e19..406af4e5f4404 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -521,6 +521,14 @@ config X86_CET_SHADOW_STACK_ALIGNMENT
 	help
 	  This option sets the alignment of the Shadow Stack.
 
+config X86_CET_IRQ_SHADOW_STACK_SIZE
+	int "IRQ Shadow Stack size"
+	default 4096
+	depends on X86_CET_SHADOW_STACK
+	help
+	  This option sets the size of the shadow stack used for interrupt
+	  and exception handling.
+
 source "arch/x86/core/Kconfig.ia32"
 source "arch/x86/core/Kconfig.intel64"
 
diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c
index 54679cdc1d5a3..a786809a1e8bc 100644
--- a/arch/x86/core/intel64/cpu.c
+++ b/arch/x86/core/intel64/cpu.c
@@ -43,6 +43,14 @@ struct x86_cpuboot x86_cpuboot[] = {
 	LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_CPU_BOOT_INIT, (,)),
 };
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_INTERRUPT_SHADOW_STACK_DEFINE, (;));
+
+struct x86_interrupt_ssp_table issp_table[] = {
+	LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_INTERRUPT_SSP_TABLE_INIT, (,)),
+};
+#endif
+
 /*
  * Send the INIT/STARTUP IPI sequence required to start up CPU 'cpu_num', which
  * will enter the kernel at fn(arg), running on the specified stack.
@@ -128,6 +136,10 @@ FUNC_NORETURN void z_x86_cpu_init(struct x86_cpuboot *cpuboot)
 #ifdef CONFIG_X86_CET_IBT
 	z_x86_ibt_enable();
 #endif /* CONFIG_X86_CET_IBT */
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	z_x86_setup_interrupt_ssp_table((uintptr_t)&issp_table[cpuboot->cpu_id]);
+#endif /* CONFIG_X86_CET_SHADOW_STACK */
+
 #endif /* CONFIG_X86_CET */
 
 	/* Enter kernel, never return */
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 3aa64e3415239..7de57a215c43b 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -863,6 +863,48 @@ irq_enter_unnested: /* Not nested: dump state to thread struct for __resume */
 	movq %rax, _thread_offset_to_ss(%rsi)
 #endif
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
+	jz __sh_stk_out_irq
+
+	/* We are on a interrupt shadow stack, which contains the three
+	 * entries needed to iretq. However, we may return to other
+	 * thread, not the one interrupted. So we need to place these
+	 * entries on the interrupted thread shadow stack, so it can be
+	 * restored later.
+	 */
+
+	rdsspq %rax
+
+	/* Load entries from current shadow stack */
+	movq (%rax), %r10 /* SSP */
+	movq 8(%rax), %r11 /* LIP */
+	movq 16(%rax), %r9 /* CS */
+
+	movq %r10, %rax /* rax now points to old shadow stack */
+
+	/* Save CS */
+	wrssq %r9, -8(%rax)
+
+	/* Save LIP */
+	wrssq %r11, -16(%rax)
+
+	/* Save SSP */
+	wrssq %r10, -24(%rax)
+
+	/* Create and "push" new token */
+	movq %rax, %r10
+	subq $24, %r10
+	orq $1, %r10
+	wrssq %r10, -32(%rax)
+
+	/* Finally, save SSP on thread struct */
+	subq $32, %rax
+	movq %rax, _thread_offset_to_shstk_addr(%rsi)
+
+__sh_stk_out_irq:
+#endif
+
 irq_dispatch:
 #ifdef CONFIG_SCHED_THREAD_USAGE
 	pushq %rcx
@@ -895,6 +937,23 @@ irq_dispatch:
 	movq ___cpu_t_current_OFFSET(%rsi), %rdi
 	call z_get_next_switch_handle
 	movq %rax, %rdi
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	/* __resume will change to next thread shadow stack (if any), so
+	 * we need to mark IRQ shadow stack as free (not busy).
+	 */
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	testl $X86_S_CET_MSR_SHSTK_EN, %eax
+	jz 1f
+
+	movq $3, %rax /* Discard SSP, LIP and CS from shadow stack */
+	incsspq %rax
+	rdsspq %rax
+	clrssbsy (%rax) /* Free shadow stack */
+1:
+#endif
+
 	jmp __resume
 
 irq_exit_nested:
diff --git a/arch/x86/include/cet.h b/arch/x86/include/cet.h
index c832f4816f807..4ecbc642e8c31 100644
--- a/arch/x86/include/cet.h
+++ b/arch/x86/include/cet.h
@@ -21,5 +21,12 @@ void z_x86_cet_enable(void);
 void z_x86_ibt_enable(void);
 #endif
 
+#if defined(CONFIG_X86_64) && defined(CONFIG_X86_CET_SHADOW_STACK)
+static inline void z_x86_setup_interrupt_ssp_table(uintptr_t issp_table)
+{
+	z_x86_msr_write(X86_INTERRUPT_SSP_TABLE_MSR, issp_table);
+}
+#endif
+
 #endif /* _ASMLANGUAGE */
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_CET_H */
diff --git a/arch/x86/include/intel64/kernel_arch_data.h b/arch/x86/include/intel64/kernel_arch_data.h
index af3b0d1c8165e..8f5595d5a8b72 100644
--- a/arch/x86/include/intel64/kernel_arch_data.h
+++ b/arch/x86/include/intel64/kernel_arch_data.h
@@ -7,6 +7,7 @@
 #define ZEPHYR_ARCH_X86_INCLUDE_INTEL64_KERNEL_ARCH_DATA_H_
 
 #include <zephyr/arch/x86/mmustructs.h>
+#include <zephyr/arch/x86/cet.h>
 
 #ifndef _ASMLANGUAGE
 
@@ -31,6 +32,17 @@ struct x86_cpuboot {
 
 typedef struct x86_cpuboot x86_cpuboot_t;
 
+struct x86_interrupt_ssp_table {
+	uintptr_t not_used;
+	uintptr_t ist1;
+	uintptr_t ist2;
+	uintptr_t ist3;
+	uintptr_t ist4;
+	uintptr_t ist5;
+	uintptr_t ist6;
+	uintptr_t ist7;
+};
+
 extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 
 #endif /* _ASMLANGUAGE */
@@ -74,6 +86,32 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 		.arg = &x86_cpu_boot_arg,							\
 	}
 
+#define X86_IRQ_SHADOW_STACK_DEFINE(name, size)							\
+	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)					\
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)					\
+	name[size / sizeof(z_x86_shadow_stack_t)] =						\
+		{ [size / sizeof(z_x86_shadow_stack_t) - 1] =					\
+			(uintptr_t)name + size - 1 * sizeof(z_x86_shadow_stack_t)		\
+		}
+
+#define X86_INTERRUPT_SHADOW_STACK_DEFINE(n, _)							\
+	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack##n,					\
+				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);			\
+	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_nmi_shadow_stack##n,					\
+				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);			\
+	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_exception_shadow_stack##n,				\
+				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE)
+
+#define SHSTK_LAST_ENTRY (CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE / sizeof(z_x86_shadow_stack_t) - 1)
+
+#define X86_INTERRUPT_SSP_TABLE_INIT(n, _)							\
+	{											\
+		.ist1 = (uintptr_t)&z_x86_irq_shadow_stack##n[SHSTK_LAST_ENTRY],		\
+		.ist6 = (uintptr_t)&z_x86_nmi_shadow_stack##n[SHSTK_LAST_ENTRY],		\
+		.ist7 = (uintptr_t)&z_x86_exception_shadow_stack##n[SHSTK_LAST_ENTRY],	\
+	}
+
+
 #define STACK_ARRAY_IDX(n, _) n
 
 #define DEFINE_STACK_ARRAY_IDX\
diff --git a/include/zephyr/arch/x86/msr.h b/include/zephyr/arch/x86/msr.h
index 9426a563f8e31..8bd7b31a7b6bd 100644
--- a/include/zephyr/arch/x86/msr.h
+++ b/include/zephyr/arch/x86/msr.h
@@ -37,6 +37,8 @@
 #define X86_S_CET_MSR_SHSTK_EN		(X86_S_CET_MSR_SHSTK | \
 					 X86_S_CET_MSR_WR_SHSTK)
 
+#define X86_INTERRUPT_SSP_TABLE_MSR	0x00006A8
+
 #define X86_EFER_MSR			0xC0000080U
 #define X86_EFER_MSR_SCE		BIT(0)
 #define X86_EFER_MSR_LME		BIT(8)

From 6bb9e179bf0356f57a051a7412c118a8f776aa7a Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Fri, 28 Mar 2025 17:22:47 -0700
Subject: [PATCH 06/21] arch/x86: Support shadow stack on nested IRQs

Nested interrupts are supported, on the normal stack, by creating a
stack whose size is a multiple of CONFIG_ISR_DEPTH, and updating the
pointer used by Interrupt Stack Table (IST) to point to a new base,
inside the "oversized" stack.

The same approach is used for the shadow stack: shadow stack size is
multiplied by CONFIG_ISR_DEPTH, and the pointer to the stack on the
shadow stack pointer table is update to point to the next base.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/intel64/cpu.c                 |  5 ++++-
 arch/x86/core/intel64/locore.S              | 10 ++++++++++
 arch/x86/core/offsets/intel64_offsets.c     |  3 +++
 arch/x86/include/intel64/kernel_arch_data.h | 15 ++++++++++-----
 include/zephyr/arch/x86/intel64/thread.h    |  2 ++
 5 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c
index a786809a1e8bc..4fdb967870637 100644
--- a/arch/x86/core/intel64/cpu.c
+++ b/arch/x86/core/intel64/cpu.c
@@ -44,7 +44,9 @@ struct x86_cpuboot x86_cpuboot[] = {
 };
 
 #ifdef CONFIG_X86_CET_SHADOW_STACK
-LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_INTERRUPT_SHADOW_STACK_DEFINE, (;));
+#define _CPU_IDX(n, _) n
+FOR_EACH(X86_INTERRUPT_SHADOW_STACK_DEFINE, (;),
+	 LISTIFY(CONFIG_MP_MAX_NUM_CPUS, _CPU_IDX, (,)));
 
 struct x86_interrupt_ssp_table issp_table[] = {
 	LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_INTERRUPT_SSP_TABLE_INIT, (,)),
@@ -138,6 +140,7 @@ FUNC_NORETURN void z_x86_cpu_init(struct x86_cpuboot *cpuboot)
 #endif /* CONFIG_X86_CET_IBT */
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 	z_x86_setup_interrupt_ssp_table((uintptr_t)&issp_table[cpuboot->cpu_id]);
+	cpuboot->gs_base->shstk_addr = &issp_table[cpuboot->cpu_id].ist1;
 #endif /* CONFIG_X86_CET_SHADOW_STACK */
 
 #endif /* CONFIG_X86_CET */
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 7de57a215c43b..73204ec6c6997 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -795,6 +795,12 @@ irq:
 
 	incl ___cpu_t_nested_OFFSET(%rsi)
 	subq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	pushq %rsi
+	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rsi
+	subq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rsi)
+	popq %rsi
+#endif
 	cmpl $CONFIG_ISR_DEPTH, ___cpu_t_nested_OFFSET(%rsi)
 	jz 1f
 	sti
@@ -929,6 +935,10 @@ irq_dispatch:
 
 	cli
 	addq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rax
+	addq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rax)
+#endif
 	decl ___cpu_t_nested_OFFSET(%rsi)
 	jnz irq_exit_nested
 
diff --git a/arch/x86/core/offsets/intel64_offsets.c b/arch/x86/core/offsets/intel64_offsets.c
index 60c772b76aa01..7365a1a8f5cdf 100644
--- a/arch/x86/core/offsets/intel64_offsets.c
+++ b/arch/x86/core/offsets/intel64_offsets.c
@@ -45,6 +45,9 @@ GEN_OFFSET_SYM(x86_tss64_t, ist2);
 GEN_OFFSET_SYM(x86_tss64_t, ist6);
 GEN_OFFSET_SYM(x86_tss64_t, ist7);
 GEN_OFFSET_SYM(x86_tss64_t, cpu);
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+GEN_OFFSET_SYM(x86_tss64_t, shstk_addr);
+#endif
 #ifdef CONFIG_USERSPACE
 GEN_OFFSET_SYM(x86_tss64_t, psp);
 GEN_OFFSET_SYM(x86_tss64_t, usp);
diff --git a/arch/x86/include/intel64/kernel_arch_data.h b/arch/x86/include/intel64/kernel_arch_data.h
index 8f5595d5a8b72..304693bcb2188 100644
--- a/arch/x86/include/intel64/kernel_arch_data.h
+++ b/arch/x86/include/intel64/kernel_arch_data.h
@@ -86,15 +86,19 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 		.arg = &x86_cpu_boot_arg,							\
 	}
 
+#define SHSTK_TOKEN_ENTRY(i, name, size)							\
+		  [size * (i + 1) / sizeof(z_x86_shadow_stack_t) - 1] =				\
+			(uintptr_t)name + size * (i + 1) - 1 * sizeof(z_x86_shadow_stack_t)	\
+
 #define X86_IRQ_SHADOW_STACK_DEFINE(name, size)							\
 	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)					\
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)					\
-	name[size / sizeof(z_x86_shadow_stack_t)] =						\
-		{ [size / sizeof(z_x86_shadow_stack_t) - 1] =					\
-			(uintptr_t)name + size - 1 * sizeof(z_x86_shadow_stack_t)		\
+	name[CONFIG_ISR_DEPTH * size / sizeof(z_x86_shadow_stack_t)] =				\
+		{										\
+			LISTIFY(CONFIG_ISR_DEPTH, SHSTK_TOKEN_ENTRY, (,), name, size)		\
 		}
 
-#define X86_INTERRUPT_SHADOW_STACK_DEFINE(n, _)							\
+#define X86_INTERRUPT_SHADOW_STACK_DEFINE(n)							\
 	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack##n,					\
 				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);			\
 	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_nmi_shadow_stack##n,					\
@@ -102,7 +106,8 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_exception_shadow_stack##n,				\
 				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE)
 
-#define SHSTK_LAST_ENTRY (CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE / sizeof(z_x86_shadow_stack_t) - 1)
+#define SHSTK_LAST_ENTRY (CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE *				\
+			  CONFIG_ISR_DEPTH / sizeof(z_x86_shadow_stack_t) - 1)
 
 #define X86_INTERRUPT_SSP_TABLE_INIT(n, _)							\
 	{											\
diff --git a/include/zephyr/arch/x86/intel64/thread.h b/include/zephyr/arch/x86/intel64/thread.h
index 5ca80131c853f..7c399914e3a75 100644
--- a/include/zephyr/arch/x86/intel64/thread.h
+++ b/include/zephyr/arch/x86/intel64/thread.h
@@ -84,6 +84,8 @@ struct x86_tss64 {
 	 */
 
 	struct _cpu *cpu;
+	uintptr_t *shstk_addr;
+
 #ifdef CONFIG_USERSPACE
 	/* Privilege mode stack pointer value when doing a system call */
 	char *psp;

From d6f5ae7f01c8d609d7dfe807bd10354558001e11 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Fri, 25 Apr 2025 13:40:26 -0700
Subject: [PATCH 07/21] arch/x86: Support shadow stack on exception handling

Main peculiarity is that if an exception results in current thread being
aborted, we need to clear the busy bit on the shadow stack on the swap
to the new thread, otherwise future exceptions will fail when trying to
use a busy shadow stack.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/intel64/cpu.c              |  1 +
 arch/x86/core/intel64/locore.S           | 11 +++++++++++
 arch/x86/core/offsets/intel64_offsets.c  |  1 +
 include/zephyr/arch/x86/intel64/thread.h |  3 +++
 4 files changed, 16 insertions(+)

diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c
index 4fdb967870637..f43688ff64822 100644
--- a/arch/x86/core/intel64/cpu.c
+++ b/arch/x86/core/intel64/cpu.c
@@ -141,6 +141,7 @@ FUNC_NORETURN void z_x86_cpu_init(struct x86_cpuboot *cpuboot)
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 	z_x86_setup_interrupt_ssp_table((uintptr_t)&issp_table[cpuboot->cpu_id]);
 	cpuboot->gs_base->shstk_addr = &issp_table[cpuboot->cpu_id].ist1;
+	cpuboot->gs_base->exception_shstk_addr = issp_table[cpuboot->cpu_id].ist7;
 #endif /* CONFIG_X86_CET_SHADOW_STACK */
 
 #endif /* CONFIG_X86_CET */
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 73204ec6c6997..c04f27426b6c4 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -378,6 +378,10 @@ z_x86_switch:
 	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
 	jz __sh_stk_out
 
+	movq %gs:(__x86_tss64_t_exception_shstk_addr_OFFSET), %rax
+	testq $1, (%rax)
+	jnz __switching_from_exception
+
 	/* Create the shadow stack restore token by hand, as using
 	 * saveprevssp is cumbersome when some threads don't use shadow
 	 * stack. Note that as we return using iretq, we also need to add
@@ -410,6 +414,13 @@ z_x86_switch:
 	movq %rcx, _thread_offset_to_shstk_addr(%rsi)
 	jmp __sh_stk_out
 
+__switching_from_exception:
+	/* We are switching from an exception, IOW, aborting current thread.
+	 * The exception shadow stack needs to be marked as not busy so that it
+	 * can be used again in case of a new exception.
+	 */
+	clrssbsy (%rax)
+
 __sh_stk_out:
 #endif
 
diff --git a/arch/x86/core/offsets/intel64_offsets.c b/arch/x86/core/offsets/intel64_offsets.c
index 7365a1a8f5cdf..d25f98f831c8a 100644
--- a/arch/x86/core/offsets/intel64_offsets.c
+++ b/arch/x86/core/offsets/intel64_offsets.c
@@ -47,6 +47,7 @@ GEN_OFFSET_SYM(x86_tss64_t, ist7);
 GEN_OFFSET_SYM(x86_tss64_t, cpu);
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 GEN_OFFSET_SYM(x86_tss64_t, shstk_addr);
+GEN_OFFSET_SYM(x86_tss64_t, exception_shstk_addr);
 #endif
 #ifdef CONFIG_USERSPACE
 GEN_OFFSET_SYM(x86_tss64_t, psp);
diff --git a/include/zephyr/arch/x86/intel64/thread.h b/include/zephyr/arch/x86/intel64/thread.h
index 7c399914e3a75..b3c54d647bc62 100644
--- a/include/zephyr/arch/x86/intel64/thread.h
+++ b/include/zephyr/arch/x86/intel64/thread.h
@@ -84,7 +84,10 @@ struct x86_tss64 {
 	 */
 
 	struct _cpu *cpu;
+#ifdef CONFIG_HW_SHADOW_STACK
 	uintptr_t *shstk_addr;
+	uintptr_t exception_shstk_addr;
+#endif
 
 #ifdef CONFIG_USERSPACE
 	/* Privilege mode stack pointer value when doing a system call */

From bf97bb297f3062a90018006339ebadc42c0a4314 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 15:04:53 -0700
Subject: [PATCH 08/21] arch/x86: Verify shadow stack pointers

Currently, it's permitted to have threads that don't have a shadow
stack. When those are run, shadow stack is disabled on the CPU. To
identify those, the thread `shstk_addr` member is checked.

This patch adds an optional check, behind
CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK, that checks if an outgoing
thread has this pointer NULL with shadow stack currently enabled on
the CPU, meaning a 1) bug or 2) some attempt to tamper with the pointer.

If the check fails, k_panic() is called. Note that this verification is
not enough to guarantee `shstk_addr` can't be tampered with. For
instance, it only works on a running thread. Ideally, all threads should
be shadow stack capable, so a missing `shstk_addr` would simply be a
hard fault, but that is still to come.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/Kconfig               |  8 ++++++++
 arch/x86/core/cet.c            | 10 +++++++++
 arch/x86/core/intel64/locore.S | 37 ++++++++++++++++++++++++++++++++--
 arch/x86/include/cet.h         |  4 ++++
 4 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 406af4e5f4404..e8808d35f3757 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -529,6 +529,14 @@ config X86_CET_IRQ_SHADOW_STACK_SIZE
 	  This option sets the size of the shadow stack used for interrupt
 	  and exception handling.
 
+config X86_CET_VERIFY_KERNEL_SHADOW_STACK
+	bool "Verify kernel shadow stack"
+	depends on X86_CET_SHADOW_STACK
+	help
+	  When enabled, when a thread is swapped out, the kernel will verify
+	  if its shadow stack pointer is not null with shadow stack enabled -
+	  which would mean some attempt to tamper with the shadow stack.
+
 source "arch/x86/core/Kconfig.ia32"
 source "arch/x86/core/Kconfig.intel64"
 
diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 9c54e7a998503..ead4d15117402 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -70,3 +70,13 @@ void z_x86_ibt_enable(void)
 	z_x86_msr_write(X86_S_CET_MSR, msr);
 }
 #endif
+
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+void z_x86_cet_shadow_stack_panic(k_tid_t *thread)
+{
+	LOG_ERR("Shadow stack enabled, but outgoing thread [%p] struct "
+		"missing shadow stack pointer", thread);
+
+	k_panic();
+}
+#endif
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index c04f27426b6c4..394ff7bbb4924 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -376,7 +376,7 @@ z_x86_switch:
 
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
-	jz __sh_stk_out
+	jz __sh_stk_verify
 
 	movq %gs:(__x86_tss64_t_exception_shstk_addr_OFFSET), %rax
 	testq $1, (%rax)
@@ -420,7 +420,21 @@ __switching_from_exception:
 	 * can be used again in case of a new exception.
 	 */
 	clrssbsy (%rax)
+	jmp __sh_stk_out
 
+__sh_stk_verify:
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+	/* Outgoing thread doesn't have a SSP, so shadow stack is disabled for
+	 *  it. Let's just check if shadow stack is enabled now, which would
+	 *  imply something is tampering with the pointer.
+	 */
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	testl $X86_S_CET_MSR_SHSTK_EN, %eax
+	jz __sh_stk_out
+	movq %rsi, %rdi
+	call z_x86_cet_shadow_stack_panic
+#endif
 __sh_stk_out:
 #endif
 
@@ -882,7 +896,7 @@ irq_enter_unnested: /* Not nested: dump state to thread struct for __resume */
 
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
-	jz __sh_stk_out_irq
+	jz __sh_stk_verify_irq
 
 	/* We are on a interrupt shadow stack, which contains the three
 	 * entries needed to iretq. However, we may return to other
@@ -919,6 +933,25 @@ irq_enter_unnested: /* Not nested: dump state to thread struct for __resume */
 	subq $32, %rax
 	movq %rax, _thread_offset_to_shstk_addr(%rsi)
 
+	jmp __sh_stk_out_irq
+
+__sh_stk_verify_irq:
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+	/* Outgoing thread doesn't have a SSP, so shadow stack is disabled for
+	 * it. Let's just check if shadow stack is enabled now, which would
+	 * imply something is tampering with the pointer.
+	 */
+
+	pushq %rcx
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	popq %rcx
+	testl $X86_S_CET_MSR_SHSTK_EN, %eax
+	jz __sh_stk_out_irq
+	movq %rsi, %rdi
+	call z_x86_cet_shadow_stack_panic
+#endif
+
 __sh_stk_out_irq:
 #endif
 
diff --git a/arch/x86/include/cet.h b/arch/x86/include/cet.h
index 4ecbc642e8c31..cd68b21d75cb6 100644
--- a/arch/x86/include/cet.h
+++ b/arch/x86/include/cet.h
@@ -28,5 +28,9 @@ static inline void z_x86_setup_interrupt_ssp_table(uintptr_t issp_table)
 }
 #endif
 
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+void z_x86_cet_shadow_stack_panic(void);
+#endif
+
 #endif /* _ASMLANGUAGE */
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_CET_H */

From de140d81b2605a0e3dd6dd2206d521896783776b Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 12:56:18 -0700
Subject: [PATCH 09/21] arch/x86: Support shadow stack on ia32

Most notable difference on base support is the need to keep the shadow
stack tokens, which are 8 bytes, 8 bytes aligned. Some helper macros are
used for that.

Also, an `ssp` entry is added to the task state segment (TSS).

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/cet.c                         |  8 +-
 arch/x86/core/fatal.c                       |  8 ++
 arch/x86/core/ia32/swap.S                   | 86 +++++++++++++++++++--
 arch/x86/core/offsets/ia32_offsets.c        |  6 ++
 arch/x86/include/cet.h                      |  1 +
 arch/x86/include/cet_asm.inc                | 40 ++++++++++
 arch/x86/include/ia32/offsets_short_arch.h  | 11 +++
 include/zephyr/arch/x86/cet.h               | 23 ++++++
 include/zephyr/arch/x86/ia32/linker.ld      | 16 ++++
 include/zephyr/arch/x86/ia32/segmentation.h |  3 +-
 include/zephyr/arch/x86/ia32/thread.h       |  6 ++
 include/zephyr/arch/x86/intel64/arch.h      |  1 +
 12 files changed, 199 insertions(+), 10 deletions(-)
 create mode 100644 arch/x86/include/cet_asm.inc

diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index ead4d15117402..95c016f6a2f08 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -18,6 +18,12 @@
 
 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
 
+#ifdef CONFIG_X86_64
+#define TOKEN_OFFSET 5
+#else
+#define TOKEN_OFFSET 4
+#endif
+
 #ifdef CONFIG_X86_CET_SHADOW_STACK
 int z_x86_thread_attach_shadow_stack(k_tid_t thread,
 				     z_x86_shadow_stack_t *stack,
@@ -36,7 +42,7 @@ int z_x86_thread_attach_shadow_stack(k_tid_t thread,
 	}
 
 	thread->arch.shstk_addr = stack + (stack_size -
-					   5 * sizeof(*stack)) / sizeof(*stack);
+					   TOKEN_OFFSET * sizeof(*stack)) / sizeof(*stack);
 	thread->arch.shstk_size = stack_size;
 	thread->arch.shstk_base = stack;
 
diff --git a/arch/x86/core/fatal.c b/arch/x86/core/fatal.c
index 6955623045b6d..4e8f25caba1a4 100644
--- a/arch/x86/core/fatal.c
+++ b/arch/x86/core/fatal.c
@@ -286,6 +286,14 @@ static void dump_regs(const struct arch_esf *esf)
 		esf->cs & 0xFFFFU, get_cr3(esf));
 
 	LOG_ERR("EIP: 0x%08x", esf->eip);
+#ifdef CONFIG_HW_SHADOW_STACK
+	{
+	uintptr_t ssp;
+
+		__asm__ volatile("rdsspd %0" : "=r"(ssp));
+		LOG_ERR("SSP: 0x%08lx", ssp);
+	}
+#endif /* CONFIG_HW_SHADOW_STACK */
 }
 #endif /* CONFIG_X86_64 */
 
diff --git a/arch/x86/core/ia32/swap.S b/arch/x86/core/ia32/swap.S
index 6103e4bbe5382..0922131eef754 100644
--- a/arch/x86/core/ia32/swap.S
+++ b/arch/x86/core/ia32/swap.S
@@ -16,6 +16,7 @@
 #include <zephyr/arch/cpu.h>
 #include <kernel_arch_data.h>
 #include <offsets_short.h>
+#include <cet_asm.inc>
 
 	/* exports (internal APIs) */
 
@@ -291,9 +292,9 @@ restoreContext_NoFloatSwap:
 	 * or XMM register.
 	 */
 
-	movl %cr0, %edx
-	orl $0x8, %edx
-	movl %edx, %cr0
+	movl %cr0, %ebx
+	orl $0x8, %ebx
+	movl %ebx, %cr0
 
 CROHandlingDone:
 
@@ -304,6 +305,7 @@ CROHandlingDone:
 	movl    %eax, _kernel_offset_to_current(%edi)
 
 #if defined(CONFIG_X86_USE_THREAD_LOCAL_STORAGE)
+	pushl   %edx
 	pushl	%eax
 
 	call	z_x86_tls_update_gdt
@@ -313,6 +315,43 @@ CROHandlingDone:
 	movw	%ax, %gs
 
 	popl	%eax
+	popl	%edx
+#endif
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	cmpl $0, _thread_offset_to_shstk_addr(%edx)
+	jz __sh_stk_verify
+
+	/* Create a ssp token "by hand", as next thread may not have shadow
+	 * stack enabled to use rstorssp/saveprevssp pair. Note that the token
+	 * has 64 bits and needs to be aligned to 8 bytes.
+	 */
+	save_ssp_restore_token %ecx, %ebx
+
+	/* Finally, save SSP on thread struct */
+	movl %ecx, _thread_offset_to_shstk_addr(%edx)
+
+	jmp __sh_stk_out
+
+__sh_stk_verify:
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+	/* Outgoing thread doesn't have a SSP, so shadow stack is disabled for
+	 * it. Let's just check if shadow stack is enabled now, which would
+	 * imply something is tampering with the pointer. */
+	pushl %eax
+	pushl %edx
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	testl $X86_S_CET_MSR_SHSTK_EN, %eax
+	jz __sh_stk_verify_out
+	/* Stack already has k_thread* */
+	call z_x86_cet_shadow_stack_panic
+
+__sh_stk_verify_out:
+	popl %edx
+	popl %eax
+#endif /* CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK */
+__sh_stk_out:
 #endif
 
 	/* recover thread stack pointer from k_thread */
@@ -339,16 +378,47 @@ CROHandlingDone:
 	 * - -EINVAL
 	 */
 
-	/* Utilize the 'eflags' parameter to arch_swap() */
-
-	pushl	4(%esp)
-	popfl
-
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	pushl %eax
+	pushl %ebx
+	pushl %edi
+
+	movl $_kernel, %eax
+	movl _kernel_offset_to_current(%eax), %edi
+	movl _thread_offset_to_shstk_addr(%edi), %ebx
+	testl %ebx, %ebx
+	jz __sh_stk_disable
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	orl $X86_S_CET_MSR_SHSTK_EN, %eax
+	wrmsr
+
+	rstorssp (%ebx)
+	discard_previous_ssp_token %ebx
+	jmp __sh_stk_done
+
+__sh_stk_disable:
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	andl $~X86_S_CET_MSR_SHSTK_EN, %eax
+	wrmsr
+
+__sh_stk_done:
+	popl %edi
+	popl %ebx
+	popl %eax
+#endif
 #if defined(CONFIG_INSTRUMENT_THREAD_SWITCHING)
 	pushl	%eax
 	call	z_thread_mark_switched_in
 	popl	%eax
 #endif
+
+	/* Utilize the 'eflags' parameter to arch_swap() */
+
+	pushl	4(%esp)
+	popfl
+
 	ret
 
 #ifdef _THREAD_WRAPPER_REQUIRED
diff --git a/arch/x86/core/offsets/ia32_offsets.c b/arch/x86/core/offsets/ia32_offsets.c
index dcbf3d7d6806e..9c5feb2154f1f 100644
--- a/arch/x86/core/offsets/ia32_offsets.c
+++ b/arch/x86/core/offsets/ia32_offsets.c
@@ -48,6 +48,12 @@ GEN_OFFSET_SYM(_thread_arch_t, ptables);
 
 GEN_OFFSET_SYM(_thread_arch_t, preempFloatReg);
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+GEN_OFFSET_SYM(_thread_arch_t, shstk_addr);
+GEN_OFFSET_SYM(_thread_arch_t, shstk_size);
+GEN_OFFSET_SYM(_thread_arch_t, shstk_base);
+#endif
+
 /**
  * size of the struct k_thread structure sans save area for floating
  * point regs
diff --git a/arch/x86/include/cet.h b/arch/x86/include/cet.h
index cd68b21d75cb6..35d53e0779bbf 100644
--- a/arch/x86/include/cet.h
+++ b/arch/x86/include/cet.h
@@ -33,4 +33,5 @@ void z_x86_cet_shadow_stack_panic(void);
 #endif
 
 #endif /* _ASMLANGUAGE */
+
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_CET_H */
diff --git a/arch/x86/include/cet_asm.inc b/arch/x86/include/cet_asm.inc
new file mode 100644
index 0000000000000..990efae8b113b
--- /dev/null
+++ b/arch/x86/include/cet_asm.inc
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2017-2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+#ifndef CONFIG_X86_64
+/* Save SSP restore on current shadow stack. It takes care of aligning the
+ * shadow stack token to 8 bytes boundary. First scratch will contain current
+ * SSP after the macro finishes.
+ * r0 - scratch register
+ * r1 - scratch register
+ */
+.macro save_ssp_restore_token r0, r1
+
+rdsspd \r0
+movl \r0, \r1
+subl $8, \r0
+andl $~0x7, \r0
+wrssd \r1, (\r0)
+xorl \r1, \r1
+wrssd \r1, 4(\r0)
+
+.endm
+
+/* Discard previous SSP token created by rstorssp. It takes care of aligning
+ * the SSP in case there was a hole in the shadow stack. Needs to be
+ * called right after rstorssp so it can use the flags set by rstorssp.
+ * r0 - scratch register
+ */
+.macro discard_previous_ssp_token r0
+movl $2, \r0
+jnc 1f
+incl \r0
+1:
+incsspd \r0
+.endm
+
+#endif /* CONFIG_X86_64 */
diff --git a/arch/x86/include/ia32/offsets_short_arch.h b/arch/x86/include/ia32/offsets_short_arch.h
index b818cf60b550b..03d5fd80c49d3 100644
--- a/arch/x86/include/ia32/offsets_short_arch.h
+++ b/arch/x86/include/ia32/offsets_short_arch.h
@@ -17,6 +17,17 @@
 #define _kernel_offset_to_fpu_owner \
 	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_fpu_owner_OFFSET)
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+#define _kernel_offset_to_shstk_addr \
+	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_shstk_addr_OFFSET)
+
+#define _kernel_offset_to_shstk_size \
+	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_shstk_size_OFFSET)
+
+#define _kernel_offset_to_shstk_base \
+	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_shstk_base_OFFSET)
+#endif
+
 /* end - kernel */
 
 /* threads */
diff --git a/include/zephyr/arch/x86/cet.h b/include/zephyr/arch/x86/cet.h
index 3b084d9c3e619..af86798dee55c 100644
--- a/include/zephyr/arch/x86/cet.h
+++ b/include/zephyr/arch/x86/cet.h
@@ -21,6 +21,7 @@ extern FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry,
 
 typedef long z_x86_shadow_stack_t;
 
+#ifdef CONFIG_X86_64
 #define Z_X86_SHADOW_STACK_DEFINE(name, size) \
 	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
@@ -33,6 +34,28 @@ typedef long z_x86_shadow_stack_t;
 		   (uintptr_t)z_thread_entry, \
 		  [size / sizeof(z_x86_shadow_stack_t) - 2] = \
 		   (uintptr_t)X86_KERNEL_CS }
+#else
+
+#ifdef CONFIG_X86_DEBUG_INFO
+extern void z_x86_thread_entry_wrapper(k_thread_entry_t entry,
+				      void *p1, void *p2, void *p3);
+#define ___x86_entry_point z_x86_thread_entry_wrapper
+#else
+#define ___x86_entry_point z_thread_entry
+#endif
+
+#define Z_X86_SHADOW_STACK_DEFINE(name, size) \
+	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
+	name[size / sizeof(z_x86_shadow_stack_t)] = \
+		{ [size / sizeof(z_x86_shadow_stack_t) - 4] = \
+		   (uintptr_t)name + size - 2 * sizeof(z_x86_shadow_stack_t), \
+		  [size / sizeof(z_x86_shadow_stack_t) - 3] = 0, \
+		  [size / sizeof(z_x86_shadow_stack_t) - 2] = \
+		   (uintptr_t)___x86_entry_point, \
+		}
+
+#endif
 
 int z_x86_thread_attach_shadow_stack(k_tid_t thread,
 				     z_x86_shadow_stack_t *stack,
diff --git a/include/zephyr/arch/x86/ia32/linker.ld b/include/zephyr/arch/x86/ia32/linker.ld
index 298e8e4f56eac..99d4cd51cb3e6 100644
--- a/include/zephyr/arch/x86/ia32/linker.ld
+++ b/include/zephyr/arch/x86/ia32/linker.ld
@@ -470,6 +470,22 @@ SECTIONS
 #include <zephyr/linker/common-ram.ld>
 #include <zephyr/arch/x86/pagetables.ld>
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+SECTION_PROLOGUE(.x86shadowstack,,)
+{
+	MMU_PAGE_ALIGN
+
+	__x86shadowstack_start = .;
+
+	*(.x86shadowstack)
+
+	MMU_PAGE_ALIGN
+
+	__x86shadowstack_end = .;
+
+} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+#endif
+
 /* Must be last in RAM */
 #include <zephyr/linker/kobject-data.ld>
 #endif /* !CONFIG_LINKER_USE_PINNED_SECTION */
diff --git a/include/zephyr/arch/x86/ia32/segmentation.h b/include/zephyr/arch/x86/ia32/segmentation.h
index 6eae975b64705..5e4617a521650 100644
--- a/include/zephyr/arch/x86/ia32/segmentation.h
+++ b/include/zephyr/arch/x86/ia32/segmentation.h
@@ -50,7 +50,7 @@ extern "C" {
 
 #ifndef _ASMLANGUAGE
 
-/* Section 7.2.1 of IA architecture SW developer manual, Vol 3. */
+/* Section 8.2.1 of IA architecture SW developer manual, Vol 3. */
 struct __packed task_state_segment {
 	uint16_t backlink;
 	uint16_t reserved_1;
@@ -91,6 +91,7 @@ struct __packed task_state_segment {
 	uint8_t t:1;		/* Trap bit */
 	uint16_t reserved_12:15;
 	uint16_t iomap;
+	uint32_t ssp;
 };
 
 #define SEG_SELECTOR(index, table, dpl) (index << 3 | table << 2 | dpl)
diff --git a/include/zephyr/arch/x86/ia32/thread.h b/include/zephyr/arch/x86/ia32/thread.h
index ad0d8f8e24e0a..7dbb61e621a38 100644
--- a/include/zephyr/arch/x86/ia32/thread.h
+++ b/include/zephyr/arch/x86/ia32/thread.h
@@ -236,6 +236,12 @@ struct _thread_arch {
 #endif /* CONFIG_LAZY_FPU_SHARING */
 
 	tPreempFloatReg preempFloatReg; /* volatile float register storage */
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	long *shstk_addr;
+	long *shstk_base;
+	size_t shstk_size;
+#endif
 };
 
 typedef struct _thread_arch _thread_arch_t;
diff --git a/include/zephyr/arch/x86/intel64/arch.h b/include/zephyr/arch/x86/intel64/arch.h
index 0376d4a90fd99..2580a768b383e 100644
--- a/include/zephyr/arch/x86/intel64/arch.h
+++ b/include/zephyr/arch/x86/intel64/arch.h
@@ -14,6 +14,7 @@
 #ifndef ZEPHYR_INCLUDE_ARCH_X86_INTEL64_ARCH_H_
 #define ZEPHYR_INCLUDE_ARCH_X86_INTEL64_ARCH_H_
 
+#include <zephyr/kernel_structs.h>
 #include <zephyr/arch/x86/intel64/exception.h>
 #include <zephyr/arch/x86/intel64/thread.h>
 #include <zephyr/arch/x86/thread_stack.h>

From 01c84a2486c3c03e9befb4c96e8418a4fef3407c Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 16:14:35 -0700
Subject: [PATCH 10/21] arch/x86: Support shadow stack on IRQ on ia32

An IRQ shadow stack is created to be used by IRQ.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/ia32/crt0.S                  |  3 +
 arch/x86/core/ia32/intstub.S               | 80 +++++++++++++++++++++-
 arch/x86/core/ia32/irq_manage.c            | 21 ++++++
 arch/x86/core/offsets/ia32_offsets.c       |  6 ++
 arch/x86/include/ia32/kernel_arch_data.h   | 11 +++
 arch/x86/include/ia32/offsets_short_arch.h | 11 +++
 include/zephyr/arch/x86/ia32/structs.h     |  9 ++-
 include/zephyr/arch/x86/intel64/arch.h     |  1 -
 8 files changed, 138 insertions(+), 4 deletions(-)

diff --git a/arch/x86/core/ia32/crt0.S b/arch/x86/core/ia32/crt0.S
index 68b1d643eef94..082da57eeccaf 100644
--- a/arch/x86/core/ia32/crt0.S
+++ b/arch/x86/core/ia32/crt0.S
@@ -283,6 +283,9 @@ __csSet:
 #ifdef CONFIG_X86_CET_IBT
 	call z_x86_ibt_enable
 #endif
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	call z_x86_set_irq_shadow_stack
+#endif
 #endif
 
 	/* load 32-bit operand size IDT */
diff --git a/arch/x86/core/ia32/intstub.S b/arch/x86/core/ia32/intstub.S
index 57c45a15f4174..780293b6deb17 100644
--- a/arch/x86/core/ia32/intstub.S
+++ b/arch/x86/core/ia32/intstub.S
@@ -19,6 +19,7 @@
 #include <offsets_short.h>
 #include <zephyr/arch/cpu.h>
 #include <zephyr/drivers/interrupt_controller/sysapic.h>
+#include <cet_asm.inc>
 
 	/* exports (internal APIs) */
 
@@ -112,12 +113,14 @@ SECTION_FUNC(PINNED_TEXT, _interrupt_enter)
 	 * EAX = isr_param, EDX = isr
 	 */
 
-	/* Push EBP as we will use it for scratch space.
-     * Also it helps in stack unwinding
+	/* Push some registers we will use it for scratch space.
+	 * Also it helps in stack unwinding
 	 * Rest of the callee-saved regs get saved by invocation of C
 	 * functions (isr handler, arch_swap(), etc)
 	 */
 	pushl	%ebp
+	pushl	%edi
+	pushl	%ebx
 
 	/* load %ecx with &_kernel */
 
@@ -144,6 +147,43 @@ SECTION_FUNC(PINNED_TEXT, _interrupt_enter)
 
 	pushl	%ebp			/* Save stack pointer */
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	movl _kernel_offset_to_current(%ecx), %ebp
+	cmpl $0, _thread_offset_to_shstk_addr(%ebp)
+	jz __sh_stk_verify
+
+	/* Create restore SSP token. Note that first scratch register will
+	 * contain SSP after this macro runs.
+	 */
+	save_ssp_restore_token %ebx, %edi
+	movl %ebx, _thread_offset_to_shstk_addr(%ebp)
+
+	movl _kernel_offset_to_shstk_addr(%ecx), %edi
+	rstorssp (%edi)
+	discard_previous_ssp_token %edi
+
+	pushl %ebx /* Save SSP */
+	jmp __sh_stk_out
+__sh_stk_verify:
+#ifdef CONFIG_X86_CET_VERIFY_KERNEL_SHADOW_STACK
+	pushl %eax
+	pushl %edx
+	pushl %ecx
+	movl $X86_S_CET_MSR, %ecx
+	rdmsr
+	testl $X86_S_CET_MSR_SHSTK_EN, %eax
+	jz __sh_stk_verify_out
+	pushl %ebp
+	call z_x86_cet_shadow_stack_panic
+
+__sh_stk_verify_out:
+	popl %ecx
+	popl %edx
+	popl %eax
+#endif
+__sh_stk_out:
+#endif
+
 #ifdef CONFIG_PM
 	cmpl	$0, _kernel_offset_to_idle(%ecx)
 	jne	handle_idle
@@ -237,6 +277,22 @@ alreadyOnIntStack:
 	 * a switch to the new thread.
 	 */
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	movl _kernel_offset_to_current(%ecx), %edi
+	cmpl $0, _thread_offset_to_shstk_addr(%edi)
+	jz __sh_stk_out_reschedule
+
+	/* No need to save SSP on _kernel as it should always be at the
+	 * same location. But we need to have a token there.
+	 */
+	save_ssp_restore_token %edi, %ebx
+
+	popl %ebp /* Get previous SSP token */
+	rstorssp (%ebp)
+	discard_previous_ssp_token %ebp
+__sh_stk_out_reschedule:
+#endif
+
 	popl	%esp	/* switch back to outgoing thread's stack */
 
 #ifdef CONFIG_STACK_SENTINEL
@@ -266,6 +322,8 @@ alreadyOnIntStack:
 #endif /* CONFIG_LAZY_FPU_SHARING */
 
 	/* Restore volatile registers and return to the interrupted thread */
+	popl	%ebx
+	popl	%edi
 	popl	%ebp
 	popl	%ecx
 	popl	%edx
@@ -283,6 +341,22 @@ noReschedule:
 	 * interrupted thread's stack and restore volatile registers
 	 */
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+	movl _kernel_offset_to_current(%ecx), %ebp
+	cmpl $0, _thread_offset_to_shstk_addr(%ebp)
+	jz __sh_stk_out_no_reschedule
+
+	/* No need to save SSP on _kernel as it should always be at the
+	 * same location. But we need to have a token there.
+	 */
+	save_ssp_restore_token %edi, %ebx
+
+	popl %ebp /* Get previous SSP token position */
+	rstorssp (%ebp)
+	discard_previous_ssp_token %ebp
+__sh_stk_out_no_reschedule:
+#endif
+
 	popl	%esp		/* pop thread stack pointer */
 
 #ifdef CONFIG_STACK_SENTINEL
@@ -299,6 +373,8 @@ noReschedule:
 	 */
 
 nestedInterrupt:
+	popl	%ebx
+	popl	%edi
 	popl	%ebp
 	popl	%ecx		/* pop volatile registers in reverse order */
 	popl	%edx
diff --git a/arch/x86/core/ia32/irq_manage.c b/arch/x86/core/ia32/irq_manage.c
index 4a3b5d03e26dc..23fcff0fe32ba 100644
--- a/arch/x86/core/ia32/irq_manage.c
+++ b/arch/x86/core/ia32/irq_manage.c
@@ -22,6 +22,14 @@
 #include <zephyr/tracing/tracing.h>
 #include <kswap.h>
 #include <zephyr/arch/x86/ia32/segmentation.h>
+#include <kernel_arch_data.h>
+
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack,
+			    CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);
+#endif
+
+#define TOKEN_OFFSET 4
 
 extern void z_SpuriousIntHandler(void *handler);
 extern void z_SpuriousIntNoErrCodeHandler(void *handler);
@@ -43,6 +51,19 @@ void arch_isr_direct_footer_swap(unsigned int key)
 	(void)z_swap_irqlock(key);
 }
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+void z_x86_set_irq_shadow_stack(void)
+{
+	size_t stack_size = CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE;
+	z_x86_shadow_stack_t *stack = z_x86_irq_shadow_stack;
+
+	_kernel.cpus[0].arch.shstk_addr = stack +
+		(stack_size - TOKEN_OFFSET * sizeof(*stack)) / sizeof(*stack);
+	_kernel.cpus[0].arch.shstk_size = stack_size;
+	_kernel.cpus[0].arch.shstk_base = stack;
+}
+#endif
+
 #if CONFIG_X86_DYNAMIC_IRQ_STUBS > 0
 
 /*
diff --git a/arch/x86/core/offsets/ia32_offsets.c b/arch/x86/core/offsets/ia32_offsets.c
index 9c5feb2154f1f..51e0429b5a349 100644
--- a/arch/x86/core/offsets/ia32_offsets.c
+++ b/arch/x86/core/offsets/ia32_offsets.c
@@ -52,6 +52,12 @@ GEN_OFFSET_SYM(_thread_arch_t, preempFloatReg);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_addr);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_size);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_base);
+
+GEN_OFFSET_SYM(_kernel_t, cpus);
+GEN_OFFSET_SYM(_cpu_t, arch);
+GEN_OFFSET_SYM(_cpu_arch_t, shstk_addr);
+GEN_OFFSET_SYM(_cpu_arch_t, shstk_size);
+GEN_OFFSET_SYM(_cpu_arch_t, shstk_base);
 #endif
 
 /**
diff --git a/arch/x86/include/ia32/kernel_arch_data.h b/arch/x86/include/ia32/kernel_arch_data.h
index f1c2943295c74..16510e51f91e4 100644
--- a/arch/x86/include/ia32/kernel_arch_data.h
+++ b/arch/x86/include/ia32/kernel_arch_data.h
@@ -30,6 +30,7 @@
 #include <zephyr/linker/sections.h>
 #include <ia32/exception.h>
 #include <zephyr/sys/util.h>
+#include <zephyr/arch/x86/cet.h>
 
 #ifndef _ASMLANGUAGE
 #include <zephyr/kernel.h>
@@ -73,4 +74,14 @@ extern void z_x86_tls_update_gdt(struct k_thread *thread);
 
 #endif /* _ASMLANGUAGE */
 
+#define X86_IRQ_SHADOW_STACK_DEFINE(name, size)						\
+	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)				\
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)				\
+	name[size / sizeof(z_x86_shadow_stack_t)] =					\
+		{ [size / sizeof(z_x86_shadow_stack_t) - 4] =				\
+			(uintptr_t)name + size - 2 * sizeof(z_x86_shadow_stack_t),	\
+		  [size / sizeof(z_x86_shadow_stack_t) - 3] = 0,			\
+		  [size / sizeof(z_x86_shadow_stack_t) - 2] = 0,			\
+		}
+
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_IA32_KERNEL_ARCH_DATA_H_ */
diff --git a/arch/x86/include/ia32/offsets_short_arch.h b/arch/x86/include/ia32/offsets_short_arch.h
index 03d5fd80c49d3..115dc70f703e5 100644
--- a/arch/x86/include/ia32/offsets_short_arch.h
+++ b/arch/x86/include/ia32/offsets_short_arch.h
@@ -41,6 +41,17 @@
 #define _thread_offset_to_preempFloatReg \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_preempFloatReg_OFFSET)
 
+#ifdef CONFIG_X86_CET_SHADOW_STACK
+#define _thread_offset_to_shstk_addr \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_addr_OFFSET)
+
+#define _thread_offset_to_shstk_size \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_size_OFFSET)
+
+#define _thread_offset_to_shstk_base \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_base_OFFSET)
+#endif
+
 /* end - threads */
 
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_IA32_OFFSETS_SHORT_ARCH_H_ */
diff --git a/include/zephyr/arch/x86/ia32/structs.h b/include/zephyr/arch/x86/ia32/structs.h
index 91112a8578c54..114e6d7241941 100644
--- a/include/zephyr/arch/x86/ia32/structs.h
+++ b/include/zephyr/arch/x86/ia32/structs.h
@@ -24,7 +24,14 @@ struct _cpu_arch {
 	 */
 
 	struct k_thread *fpu_owner;
-#elif defined(__cplusplus)
+#endif
+#if defined(CONFIG_X86_CET_SHADOW_STACK)
+	long *shstk_addr; /* Latest top of shadow stack */
+	long *shstk_base; /* Base of shadow stack */
+	size_t shstk_size;
+#endif
+#if defined(__cplusplus) && !defined(CONFIG_FPU_SHARING) && \
+		!defined(CONFIG_X86_CET_SHADOW_STACK)
 	/* Ensure this struct does not have a size of 0 which is not allowed in C++. */
 	uint8_t dummy;
 #endif
diff --git a/include/zephyr/arch/x86/intel64/arch.h b/include/zephyr/arch/x86/intel64/arch.h
index 2580a768b383e..0376d4a90fd99 100644
--- a/include/zephyr/arch/x86/intel64/arch.h
+++ b/include/zephyr/arch/x86/intel64/arch.h
@@ -14,7 +14,6 @@
 #ifndef ZEPHYR_INCLUDE_ARCH_X86_INTEL64_ARCH_H_
 #define ZEPHYR_INCLUDE_ARCH_X86_INTEL64_ARCH_H_
 
-#include <zephyr/kernel_structs.h>
 #include <zephyr/arch/x86/intel64/exception.h>
 #include <zephyr/arch/x86/intel64/thread.h>
 #include <zephyr/arch/x86/thread_stack.h>

From 0fd830225095a63ef772cc504ab39a967c7bf5a5 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 25 Mar 2025 15:55:31 -0700
Subject: [PATCH 11/21] arch/x86: Allow SoC to run preparatory steps on shadow
 stack

Some SoCs may need to do some preparatory work before changing the
current shadow stack pointer (and thus, currently used shadow stack).
This patch adds a way for that, shielded by a Kconfig
(CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH).

As currently only 32 bit SoC may use this, support is only added to the
32 bit code.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/Kconfig             |  8 ++++++++
 arch/x86/core/ia32/intstub.S | 32 ++++++++++++++++++++++++++++++--
 arch/x86/core/ia32/swap.S    | 10 ++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e8808d35f3757..3ebca62444a5e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -537,6 +537,14 @@ config X86_CET_VERIFY_KERNEL_SHADOW_STACK
 	  if its shadow stack pointer is not null with shadow stack enabled -
 	  which would mean some attempt to tamper with the shadow stack.
 
+config X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	bool "SOC prepare shadow stack switch"
+	depends on X86_CET_SHADOW_STACK
+	help
+	  When enabled, call SoC code just before switching shadow stacks.
+	  Code should be provided via soc_prepare_shadow_stack_switch assembler
+	  macro.
+
 source "arch/x86/core/Kconfig.ia32"
 source "arch/x86/core/Kconfig.intel64"
 
diff --git a/arch/x86/core/ia32/intstub.S b/arch/x86/core/ia32/intstub.S
index 780293b6deb17..6157fcc7a998f 100644
--- a/arch/x86/core/ia32/intstub.S
+++ b/arch/x86/core/ia32/intstub.S
@@ -21,6 +21,10 @@
 #include <zephyr/drivers/interrupt_controller/sysapic.h>
 #include <cet_asm.inc>
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+#include <soc_cet.h>
+#endif
+
 	/* exports (internal APIs) */
 
 	GTEXT(_interrupt_enter)
@@ -152,12 +156,24 @@ SECTION_FUNC(PINNED_TEXT, _interrupt_enter)
 	cmpl $0, _thread_offset_to_shstk_addr(%ebp)
 	jz __sh_stk_verify
 
-	/* Create restore SSP token. Note that first scratch register will
-	 * contain SSP after this macro runs.
+	/* Create restore SSP token. We can't use saveprevssp as it may
+	 * conflict with the prepare_shadow_stack_switch from an SoC.
+	 * Note that first scratch register will contain SSP after this
+	 * macro runs.
 	 */
 	save_ssp_restore_token %ebx, %edi
 	movl %ebx, _thread_offset_to_shstk_addr(%ebp)
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	pushl %eax
+	pushl %edx
+	pushl %ecx
+	soc_prepare_irq_shadow_stack_switch
+	popl %ecx
+	popl %edx
+	popl %eax
+#endif
+
 	movl _kernel_offset_to_shstk_addr(%ecx), %edi
 	rstorssp (%edi)
 	discard_previous_ssp_token %edi
@@ -287,6 +303,12 @@ alreadyOnIntStack:
 	 */
 	save_ssp_restore_token %edi, %ebx
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	pushl %edi
+	soc_prepare_shadow_stack_switch
+	popl %edi
+#endif
+
 	popl %ebp /* Get previous SSP token */
 	rstorssp (%ebp)
 	discard_previous_ssp_token %ebp
@@ -351,6 +373,12 @@ noReschedule:
 	 */
 	save_ssp_restore_token %edi, %ebx
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	pushl %ebp
+	soc_prepare_shadow_stack_switch
+	popl %ebp
+#endif
+
 	popl %ebp /* Get previous SSP token position */
 	rstorssp (%ebp)
 	discard_previous_ssp_token %ebp
diff --git a/arch/x86/core/ia32/swap.S b/arch/x86/core/ia32/swap.S
index 0922131eef754..361f7101fb27a 100644
--- a/arch/x86/core/ia32/swap.S
+++ b/arch/x86/core/ia32/swap.S
@@ -18,6 +18,10 @@
 #include <offsets_short.h>
 #include <cet_asm.inc>
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+#include <soc_cet.h>
+#endif
+
 	/* exports (internal APIs) */
 
 	GTEXT(arch_swap)
@@ -393,6 +397,12 @@ __sh_stk_out:
 	orl $X86_S_CET_MSR_SHSTK_EN, %eax
 	wrmsr
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	pushl %edi
+	soc_prepare_shadow_stack_switch
+	popl %edi
+#endif
+
 	rstorssp (%ebx)
 	discard_previous_ssp_token %ebx
 	jmp __sh_stk_done

From e340b4fe24633ef1404df8b9b43e6fb6d1330689 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 1 Apr 2025 18:01:10 -0700
Subject: [PATCH 12/21] kernel: Add interface for hardware shadow stack usage

In order to allow kernel created threads (such as main and idle threads)
to make use of hardware shadow stack implementation, add an interface
for them.

This patch basically provides an infra that architectures need to
implement to provide hardware shadow stack.

Also, main and idle threads are updated to make use of this interface
(if hardware shadow stacks are enabled).

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/Kconfig                         |  3 +++
 include/zephyr/kernel/thread_stack.h |  8 ++++++++
 kernel/Kconfig                       | 24 ++++++++++++++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/arch/Kconfig b/arch/Kconfig
index d059e2a2a221f..9272e8456126e 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -763,6 +763,9 @@ config ARCH_HAS_THREAD_PRIV_STACK_SPACE_GET
 	help
 	  Select when the architecture implements arch_thread_priv_stack_space_get().
 
+config ARCH_HAS_HW_SHADOW_STACK
+	bool
+
 #
 # Other architecture related options
 #
diff --git a/include/zephyr/kernel/thread_stack.h b/include/zephyr/kernel/thread_stack.h
index 2989624e04367..4f41de2580d06 100644
--- a/include/zephyr/kernel/thread_stack.h
+++ b/include/zephyr/kernel/thread_stack.h
@@ -283,6 +283,14 @@ static inline char *z_stack_ptr_align(char *ptr)
 
 /** @} */
 
+#ifdef CONFIG_HW_SHADOW_STACK
+#define K_THREAD_HW_SHADOW_STACK_DEFINE ARCH_THREAD_HW_SHADOW_STACK_DEFINE
+
+#define k_thread_hw_shadow_stack_t arch_thread_hw_shadow_stack_t
+
+#define k_thread_hw_shadow_stack_attach arch_thread_hw_shadow_stack_attach
+#endif
+
 static inline char *K_KERNEL_STACK_BUFFER(k_thread_stack_t *sym)
 {
 	return (char *)sym + K_KERNEL_STACK_RESERVED;
diff --git a/kernel/Kconfig b/kernel/Kconfig
index 9098f7da050f7..c7ed58b7f1025 100644
--- a/kernel/Kconfig
+++ b/kernel/Kconfig
@@ -1029,6 +1029,30 @@ config BOUNDS_CHECK_BYPASS_MITIGATION
 	  mitigations after bounds checking any array index parameters passed
 	  in from untrusted sources (user mode threads). When disabled, these
 	  macros do nothing.
+
+config HW_SHADOW_STACK
+	bool "Use hardware shadow stack mechanism to provide stack protection [EXPERIMENTAL]"
+	depends on ARCH_HAS_HW_SHADOW_STACK
+	select EXPERIMENTAL
+	help
+	  This option enables the use of hardware shadow stack to provide
+	  stack protection. When enabled, threads that provide a shadow stack
+	  will use the hardware shadow stack to store the return address of
+	  function calls. This will help to protect against return-oriented
+	  programming (ROP) attacks.
+
+	  Currently, not all threads created by the kernel will have a shadow
+	  stack. Indeed, threads need to manually attach a shadow stack to
+	  use this feature. See the documentation for more information.
+
+config KERNEL_HW_SHADOW_STACK_SIZE
+	int "Size of the hardware shadow stack"
+	depends on HW_SHADOW_STACK
+	default 1024
+	help
+	  This option specifies the size of the hardware shadow stack, in bytes,
+	  for kernel created threads, such as main and idle threads.
+
 endmenu
 
 rsource "Kconfig.mem_domain"

From b3483cca07b9028d789d2f478fcd26a261088aa4 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Wed, 7 May 2025 14:02:34 -0700
Subject: [PATCH 13/21] arch/x86: Use Zephyr HW shadow stack arch interface

So that kernel created threads can use shadow stacks. Note that
CONFIG_X86_CET_SHADOW_STACK is abandoned in favour of
CONFIG_HW_SHADOW_STACK.

This means change some types, functions and macro throughout shadow
stack code.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/Kconfig                              | 16 ++----
 arch/x86/core/cet.c                           |  8 +--
 arch/x86/core/ia32/crt0.S                     |  2 +-
 arch/x86/core/ia32/intstub.S                  |  6 +--
 arch/x86/core/ia32/irq_manage.c               |  6 +--
 arch/x86/core/ia32/swap.S                     |  4 +-
 arch/x86/core/intel64/cpu.c                   |  6 +--
 arch/x86/core/intel64/locore.S                | 12 ++---
 arch/x86/core/offsets/ia32_offsets.c          |  2 +-
 arch/x86/core/offsets/intel64_offsets.c       |  4 +-
 arch/x86/gen_mmu.py                           |  2 +-
 arch/x86/include/cet.h                        |  2 +-
 arch/x86/include/ia32/kernel_arch_data.h      | 16 +++---
 arch/x86/include/ia32/offsets_short_arch.h    |  4 +-
 arch/x86/include/intel64/kernel_arch_data.h   | 11 ++--
 arch/x86/include/intel64/offsets_short_arch.h |  2 +-
 include/zephyr/arch/x86/arch.h                |  1 +
 include/zephyr/arch/x86/cet.h                 | 50 +++++++++----------
 include/zephyr/arch/x86/ia32/linker.ld        |  2 +-
 include/zephyr/arch/x86/ia32/structs.h        |  4 +-
 include/zephyr/arch/x86/ia32/thread.h         |  2 +-
 include/zephyr/arch/x86/intel64/linker.ld     |  2 +-
 include/zephyr/arch/x86/intel64/thread.h      |  2 +-
 23 files changed, 79 insertions(+), 87 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3ebca62444a5e..b8b8f08575b38 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -112,6 +112,7 @@ config X86_CPU_HAS_SSE4A
 	bool
 
 config X86_CPU_HAS_CET
+	select ARCH_HAS_HW_SHADOW_STACK if !USERSPACE && !DYNAMIC_THREAD_ALLOC
 	bool
 
 if FPU || X86_64
@@ -507,31 +508,24 @@ config X86_CET_IBT
 	help
 	  This option enables Indirect Branch Tracking (IBT) support.
 
-config X86_CET_SHADOW_STACK
-	bool "Shadow Stack"
-	depends on X86_CET
-	depends on SRAM_REGION_PERMISSIONS
-	help
-	  This option enables Shadow Stack support.
-
 config X86_CET_SHADOW_STACK_ALIGNMENT
 	int "Shadow Stack alignment"
 	default 4096
-	depends on X86_CET_SHADOW_STACK
+	depends on HW_SHADOW_STACK
 	help
 	  This option sets the alignment of the Shadow Stack.
 
 config X86_CET_IRQ_SHADOW_STACK_SIZE
 	int "IRQ Shadow Stack size"
 	default 4096
-	depends on X86_CET_SHADOW_STACK
+	depends on HW_SHADOW_STACK
 	help
 	  This option sets the size of the shadow stack used for interrupt
 	  and exception handling.
 
 config X86_CET_VERIFY_KERNEL_SHADOW_STACK
 	bool "Verify kernel shadow stack"
-	depends on X86_CET_SHADOW_STACK
+	depends on HW_SHADOW_STACK
 	help
 	  When enabled, when a thread is swapped out, the kernel will verify
 	  if its shadow stack pointer is not null with shadow stack enabled -
@@ -539,7 +533,7 @@ config X86_CET_VERIFY_KERNEL_SHADOW_STACK
 
 config X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
 	bool "SOC prepare shadow stack switch"
-	depends on X86_CET_SHADOW_STACK
+	depends on HW_SHADOW_STACK
 	help
 	  When enabled, call SoC code just before switching shadow stacks.
 	  Code should be provided via soc_prepare_shadow_stack_switch assembler
diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 95c016f6a2f08..3d2225f9c32df 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -24,10 +24,10 @@ LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
 #define TOKEN_OFFSET 4
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
-int z_x86_thread_attach_shadow_stack(k_tid_t thread,
-				     z_x86_shadow_stack_t *stack,
-				     size_t stack_size)
+#ifdef CONFIG_HW_SHADOW_STACK
+int arch_thread_hw_shadow_stack_attach(k_tid_t thread,
+				       arch_thread_hw_shadow_stack_t *stack,
+				       size_t stack_size)
 {
 	/* Can't attach to NULL */
 	if (stack == NULL) {
diff --git a/arch/x86/core/ia32/crt0.S b/arch/x86/core/ia32/crt0.S
index 082da57eeccaf..59c4a587a714c 100644
--- a/arch/x86/core/ia32/crt0.S
+++ b/arch/x86/core/ia32/crt0.S
@@ -283,7 +283,7 @@ __csSet:
 #ifdef CONFIG_X86_CET_IBT
 	call z_x86_ibt_enable
 #endif
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	call z_x86_set_irq_shadow_stack
 #endif
 #endif
diff --git a/arch/x86/core/ia32/intstub.S b/arch/x86/core/ia32/intstub.S
index 6157fcc7a998f..d1daf81223224 100644
--- a/arch/x86/core/ia32/intstub.S
+++ b/arch/x86/core/ia32/intstub.S
@@ -151,7 +151,7 @@ SECTION_FUNC(PINNED_TEXT, _interrupt_enter)
 
 	pushl	%ebp			/* Save stack pointer */
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	movl _kernel_offset_to_current(%ecx), %ebp
 	cmpl $0, _thread_offset_to_shstk_addr(%ebp)
 	jz __sh_stk_verify
@@ -293,7 +293,7 @@ alreadyOnIntStack:
 	 * a switch to the new thread.
 	 */
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	movl _kernel_offset_to_current(%ecx), %edi
 	cmpl $0, _thread_offset_to_shstk_addr(%edi)
 	jz __sh_stk_out_reschedule
@@ -363,7 +363,7 @@ noReschedule:
 	 * interrupted thread's stack and restore volatile registers
 	 */
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	movl _kernel_offset_to_current(%ecx), %ebp
 	cmpl $0, _thread_offset_to_shstk_addr(%ebp)
 	jz __sh_stk_out_no_reschedule
diff --git a/arch/x86/core/ia32/irq_manage.c b/arch/x86/core/ia32/irq_manage.c
index 23fcff0fe32ba..bb7f0b6397b47 100644
--- a/arch/x86/core/ia32/irq_manage.c
+++ b/arch/x86/core/ia32/irq_manage.c
@@ -24,7 +24,7 @@
 #include <zephyr/arch/x86/ia32/segmentation.h>
 #include <kernel_arch_data.h>
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack,
 			    CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);
 #endif
@@ -51,11 +51,11 @@ void arch_isr_direct_footer_swap(unsigned int key)
 	(void)z_swap_irqlock(key);
 }
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 void z_x86_set_irq_shadow_stack(void)
 {
 	size_t stack_size = CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE;
-	z_x86_shadow_stack_t *stack = z_x86_irq_shadow_stack;
+	arch_thread_hw_shadow_stack_t *stack = z_x86_irq_shadow_stack;
 
 	_kernel.cpus[0].arch.shstk_addr = stack +
 		(stack_size - TOKEN_OFFSET * sizeof(*stack)) / sizeof(*stack);
diff --git a/arch/x86/core/ia32/swap.S b/arch/x86/core/ia32/swap.S
index 361f7101fb27a..ff01e6b4a2ef1 100644
--- a/arch/x86/core/ia32/swap.S
+++ b/arch/x86/core/ia32/swap.S
@@ -322,7 +322,7 @@ CROHandlingDone:
 	popl	%edx
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	cmpl $0, _thread_offset_to_shstk_addr(%edx)
 	jz __sh_stk_verify
 
@@ -382,7 +382,7 @@ __sh_stk_out:
 	 * - -EINVAL
 	 */
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	pushl %eax
 	pushl %ebx
 	pushl %edi
diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c
index f43688ff64822..69bbb91466fbb 100644
--- a/arch/x86/core/intel64/cpu.c
+++ b/arch/x86/core/intel64/cpu.c
@@ -43,7 +43,7 @@ struct x86_cpuboot x86_cpuboot[] = {
 	LISTIFY(CONFIG_MP_MAX_NUM_CPUS, X86_CPU_BOOT_INIT, (,)),
 };
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 #define _CPU_IDX(n, _) n
 FOR_EACH(X86_INTERRUPT_SHADOW_STACK_DEFINE, (;),
 	 LISTIFY(CONFIG_MP_MAX_NUM_CPUS, _CPU_IDX, (,)));
@@ -138,11 +138,11 @@ FUNC_NORETURN void z_x86_cpu_init(struct x86_cpuboot *cpuboot)
 #ifdef CONFIG_X86_CET_IBT
 	z_x86_ibt_enable();
 #endif /* CONFIG_X86_CET_IBT */
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	z_x86_setup_interrupt_ssp_table((uintptr_t)&issp_table[cpuboot->cpu_id]);
 	cpuboot->gs_base->shstk_addr = &issp_table[cpuboot->cpu_id].ist1;
 	cpuboot->gs_base->exception_shstk_addr = issp_table[cpuboot->cpu_id].ist7;
-#endif /* CONFIG_X86_CET_SHADOW_STACK */
+#endif /* CONFIG_HW_SHADOW_STACK */
 
 #endif /* CONFIG_X86_CET */
 
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 394ff7bbb4924..439fea6619c9e 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -374,7 +374,7 @@ z_x86_switch:
 	movq $X86_KERNEL_DS, _thread_offset_to_ss(%rsi)
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
 	jz __sh_stk_verify
 
@@ -473,7 +473,7 @@ __resume:
 	wrmsr
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	movq _thread_offset_to_shstk_addr(%rdi), %r10
 	testq %r10, %r10
 	jz __sh_stk_disable
@@ -820,7 +820,7 @@ irq:
 
 	incl ___cpu_t_nested_OFFSET(%rsi)
 	subq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	pushq %rsi
 	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rsi
 	subq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rsi)
@@ -894,7 +894,7 @@ irq_enter_unnested: /* Not nested: dump state to thread struct for __resume */
 	movq %rax, _thread_offset_to_ss(%rsi)
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	cmpq $0, _thread_offset_to_shstk_addr(%rsi)
 	jz __sh_stk_verify_irq
 
@@ -979,7 +979,7 @@ irq_dispatch:
 
 	cli
 	addq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rax
 	addq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rax)
 #endif
@@ -992,7 +992,7 @@ irq_dispatch:
 	call z_get_next_switch_handle
 	movq %rax, %rdi
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	/* __resume will change to next thread shadow stack (if any), so
 	 * we need to mark IRQ shadow stack as free (not busy).
 	 */
diff --git a/arch/x86/core/offsets/ia32_offsets.c b/arch/x86/core/offsets/ia32_offsets.c
index 51e0429b5a349..d08db0cf06b57 100644
--- a/arch/x86/core/offsets/ia32_offsets.c
+++ b/arch/x86/core/offsets/ia32_offsets.c
@@ -48,7 +48,7 @@ GEN_OFFSET_SYM(_thread_arch_t, ptables);
 
 GEN_OFFSET_SYM(_thread_arch_t, preempFloatReg);
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 GEN_OFFSET_SYM(_thread_arch_t, shstk_addr);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_size);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_base);
diff --git a/arch/x86/core/offsets/intel64_offsets.c b/arch/x86/core/offsets/intel64_offsets.c
index d25f98f831c8a..61bf0c282387c 100644
--- a/arch/x86/core/offsets/intel64_offsets.c
+++ b/arch/x86/core/offsets/intel64_offsets.c
@@ -34,7 +34,7 @@ GEN_OFFSET_SYM(_thread_arch_t, psp);
 GEN_OFFSET_SYM(_thread_arch_t, ptables);
 #endif
 #endif /* CONFIG_USERSPACE */
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 GEN_OFFSET_SYM(_thread_arch_t, shstk_addr);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_base);
 GEN_OFFSET_SYM(_thread_arch_t, shstk_size);
@@ -45,7 +45,7 @@ GEN_OFFSET_SYM(x86_tss64_t, ist2);
 GEN_OFFSET_SYM(x86_tss64_t, ist6);
 GEN_OFFSET_SYM(x86_tss64_t, ist7);
 GEN_OFFSET_SYM(x86_tss64_t, cpu);
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 GEN_OFFSET_SYM(x86_tss64_t, shstk_addr);
 GEN_OFFSET_SYM(x86_tss64_t, exception_shstk_addr);
 #endif
diff --git a/arch/x86/gen_mmu.py b/arch/x86/gen_mmu.py
index 8bfe2501e5f72..638f13356d0fa 100755
--- a/arch/x86/gen_mmu.py
+++ b/arch/x86/gen_mmu.py
@@ -913,7 +913,7 @@ def main():
             pt.set_region_perms("_locore", FLAG_P | flag_user)
             pt.set_region_perms("_lorodata", FLAG_P | ENTRY_XD | flag_user)
 
-        if isdef("CONFIG_X86_CET_SHADOW_STACK"):
+        if isdef("CONFIG_HW_SHADOW_STACK"):
             pt.set_region_perms("__x86shadowstack", FLAG_P | FLAG_D | ENTRY_XD)
 
     written_size = pt.write_output(args.output)
diff --git a/arch/x86/include/cet.h b/arch/x86/include/cet.h
index 35d53e0779bbf..7713081bfc437 100644
--- a/arch/x86/include/cet.h
+++ b/arch/x86/include/cet.h
@@ -21,7 +21,7 @@ void z_x86_cet_enable(void);
 void z_x86_ibt_enable(void);
 #endif
 
-#if defined(CONFIG_X86_64) && defined(CONFIG_X86_CET_SHADOW_STACK)
+#if defined(CONFIG_X86_64) && defined(CONFIG_HW_SHADOW_STACK)
 static inline void z_x86_setup_interrupt_ssp_table(uintptr_t issp_table)
 {
 	z_x86_msr_write(X86_INTERRUPT_SSP_TABLE_MSR, issp_table);
diff --git a/arch/x86/include/ia32/kernel_arch_data.h b/arch/x86/include/ia32/kernel_arch_data.h
index 16510e51f91e4..2210d700f2301 100644
--- a/arch/x86/include/ia32/kernel_arch_data.h
+++ b/arch/x86/include/ia32/kernel_arch_data.h
@@ -74,14 +74,14 @@ extern void z_x86_tls_update_gdt(struct k_thread *thread);
 
 #endif /* _ASMLANGUAGE */
 
-#define X86_IRQ_SHADOW_STACK_DEFINE(name, size)						\
-	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)				\
-	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)				\
-	name[size / sizeof(z_x86_shadow_stack_t)] =					\
-		{ [size / sizeof(z_x86_shadow_stack_t) - 4] =				\
-			(uintptr_t)name + size - 2 * sizeof(z_x86_shadow_stack_t),	\
-		  [size / sizeof(z_x86_shadow_stack_t) - 3] = 0,			\
-		  [size / sizeof(z_x86_shadow_stack_t) - 2] = 0,			\
+#define X86_IRQ_SHADOW_STACK_DEFINE(name, size)							\
+	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)			\
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)					\
+	name[size / sizeof(arch_thread_hw_shadow_stack_t)] =					\
+		{ [size / sizeof(arch_thread_hw_shadow_stack_t) - 4] =				\
+			(uintptr_t)name + size - 2 * sizeof(arch_thread_hw_shadow_stack_t),	\
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 3] = 0,			\
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 2] = 0,			\
 		}
 
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_IA32_KERNEL_ARCH_DATA_H_ */
diff --git a/arch/x86/include/ia32/offsets_short_arch.h b/arch/x86/include/ia32/offsets_short_arch.h
index 115dc70f703e5..2497a659a9080 100644
--- a/arch/x86/include/ia32/offsets_short_arch.h
+++ b/arch/x86/include/ia32/offsets_short_arch.h
@@ -17,7 +17,7 @@
 #define _kernel_offset_to_fpu_owner \
 	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_fpu_owner_OFFSET)
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 #define _kernel_offset_to_shstk_addr \
 	(___kernel_t_cpus_OFFSET + ___cpu_t_arch_OFFSET + ___cpu_arch_t_shstk_addr_OFFSET)
 
@@ -41,7 +41,7 @@
 #define _thread_offset_to_preempFloatReg \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_preempFloatReg_OFFSET)
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 #define _thread_offset_to_shstk_addr \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_addr_OFFSET)
 
diff --git a/arch/x86/include/intel64/kernel_arch_data.h b/arch/x86/include/intel64/kernel_arch_data.h
index 304693bcb2188..da24fc65a791f 100644
--- a/arch/x86/include/intel64/kernel_arch_data.h
+++ b/arch/x86/include/intel64/kernel_arch_data.h
@@ -87,13 +87,14 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 	}
 
 #define SHSTK_TOKEN_ENTRY(i, name, size)							\
-		  [size * (i + 1) / sizeof(z_x86_shadow_stack_t) - 1] =				\
-			(uintptr_t)name + size * (i + 1) - 1 * sizeof(z_x86_shadow_stack_t)	\
+		  [size * (i + 1) / sizeof(arch_thread_hw_shadow_stack_t) - 1] =		\
+			(uintptr_t)name + size * (i + 1) - 1 *					\
+			sizeof(arch_thread_hw_shadow_stack_t)
 
 #define X86_IRQ_SHADOW_STACK_DEFINE(name, size)							\
-	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)					\
+	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)			\
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)					\
-	name[CONFIG_ISR_DEPTH * size / sizeof(z_x86_shadow_stack_t)] =				\
+	name[CONFIG_ISR_DEPTH * size / sizeof(arch_thread_hw_shadow_stack_t)] =			\
 		{										\
 			LISTIFY(CONFIG_ISR_DEPTH, SHSTK_TOKEN_ENTRY, (,), name, size)		\
 		}
@@ -107,7 +108,7 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE)
 
 #define SHSTK_LAST_ENTRY (CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE *				\
-			  CONFIG_ISR_DEPTH / sizeof(z_x86_shadow_stack_t) - 1)
+			  CONFIG_ISR_DEPTH / sizeof(arch_thread_hw_shadow_stack_t) - 1)
 
 #define X86_INTERRUPT_SSP_TABLE_INIT(n, _)							\
 	{											\
diff --git a/arch/x86/include/intel64/offsets_short_arch.h b/arch/x86/include/intel64/offsets_short_arch.h
index 6a8ab1c193a2c..1637906067206 100644
--- a/arch/x86/include/intel64/offsets_short_arch.h
+++ b/arch/x86/include/intel64/offsets_short_arch.h
@@ -71,7 +71,7 @@
 #define _thread_offset_to_cs \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_cs_OFFSET)
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 #define _thread_offset_to_shstk_addr \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_shstk_addr_OFFSET)
 
diff --git a/include/zephyr/arch/x86/arch.h b/include/zephyr/arch/x86/arch.h
index adb8d06fa6ccf..29a2932340f3c 100644
--- a/include/zephyr/arch/x86/arch.h
+++ b/include/zephyr/arch/x86/arch.h
@@ -31,6 +31,7 @@
 #include <zephyr/irq.h>
 #include <zephyr/arch/x86/mmustructs.h>
 #include <zephyr/arch/x86/thread_stack.h>
+#include <zephyr/arch/x86/cet.h>
 #include <zephyr/linker/sections.h>
 
 #ifdef __cplusplus
diff --git a/include/zephyr/arch/x86/cet.h b/include/zephyr/arch/x86/cet.h
index af86798dee55c..fd366c4286548 100644
--- a/include/zephyr/arch/x86/cet.h
+++ b/include/zephyr/arch/x86/cet.h
@@ -8,31 +8,27 @@
 #ifndef ZEPHYR_INCLUDE_ARCH_X86_CET_H_
 #define ZEPHYR_INCLUDE_ARCH_X86_CET_H_
 
-#include <zephyr/kernel.h>
-#include <zephyr/toolchain.h>
-#include <zephyr/sys/util.h>
-
 #ifndef _ASMLANGUAGE
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 
 extern FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry,
 					 void *p1, void *p2, void *p3);
 
-typedef long z_x86_shadow_stack_t;
+typedef uintptr_t arch_thread_hw_shadow_stack_t;
 
 #ifdef CONFIG_X86_64
-#define Z_X86_SHADOW_STACK_DEFINE(name, size) \
-	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
+#define ARCH_THREAD_HW_SHADOW_STACK_DEFINE(name, size) \
+	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
-	name[size / sizeof(z_x86_shadow_stack_t)] = \
-		{ [size / sizeof(z_x86_shadow_stack_t) - 5] = \
-		   (uintptr_t)name + size - 4 * sizeof(z_x86_shadow_stack_t) + 1, \
-		  [size / sizeof(z_x86_shadow_stack_t) - 4] = \
-		  (uintptr_t)name + size - 1 * sizeof(z_x86_shadow_stack_t), \
-		  [size / sizeof(z_x86_shadow_stack_t) - 3] = \
+	name[size / sizeof(arch_thread_hw_shadow_stack_t)] = \
+		{ [size / sizeof(arch_thread_hw_shadow_stack_t) - 5] = \
+		   (uintptr_t)name + size - 4 * sizeof(arch_thread_hw_shadow_stack_t) + 1, \
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 4] = \
+		  (uintptr_t)name + size - 1 * sizeof(arch_thread_hw_shadow_stack_t), \
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 3] = \
 		   (uintptr_t)z_thread_entry, \
-		  [size / sizeof(z_x86_shadow_stack_t) - 2] = \
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 2] = \
 		   (uintptr_t)X86_KERNEL_CS }
 #else
 
@@ -44,24 +40,24 @@ extern void z_x86_thread_entry_wrapper(k_thread_entry_t entry,
 #define ___x86_entry_point z_thread_entry
 #endif
 
-#define Z_X86_SHADOW_STACK_DEFINE(name, size) \
-	z_x86_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
+#define ARCH_THREAD_HW_SHADOW_STACK_DEFINE(name, size) \
+	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
-	name[size / sizeof(z_x86_shadow_stack_t)] = \
-		{ [size / sizeof(z_x86_shadow_stack_t) - 4] = \
-		   (uintptr_t)name + size - 2 * sizeof(z_x86_shadow_stack_t), \
-		  [size / sizeof(z_x86_shadow_stack_t) - 3] = 0, \
-		  [size / sizeof(z_x86_shadow_stack_t) - 2] = \
+	name[size / sizeof(arch_thread_hw_shadow_stack_t)] = \
+		{ [size / sizeof(arch_thread_hw_shadow_stack_t) - 4] = \
+		   (uintptr_t)name + size - 2 * sizeof(arch_thread_hw_shadow_stack_t), \
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 3] = 0, \
+		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 2] = \
 		   (uintptr_t)___x86_entry_point, \
 		}
 
-#endif
+#endif /* CONFIG_X86_64 */
 
-int z_x86_thread_attach_shadow_stack(k_tid_t thread,
-				     z_x86_shadow_stack_t *stack,
-				     size_t stack_size);
+int arch_thread_hw_shadow_stack_attach(struct k_thread *thread,
+				       arch_thread_hw_shadow_stack_t *stack,
+				       size_t stack_size);
 
-#endif
+#endif /* CONFIG_HW_SHADOW_STACK */
 
 #endif /* _ASMLANGUAGE */
 #endif /* ZEPHYR_INCLUDE_ARCH_X86_CET_H_ */
diff --git a/include/zephyr/arch/x86/ia32/linker.ld b/include/zephyr/arch/x86/ia32/linker.ld
index 99d4cd51cb3e6..6bdf481e1968e 100644
--- a/include/zephyr/arch/x86/ia32/linker.ld
+++ b/include/zephyr/arch/x86/ia32/linker.ld
@@ -470,7 +470,7 @@ SECTIONS
 #include <zephyr/linker/common-ram.ld>
 #include <zephyr/arch/x86/pagetables.ld>
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 SECTION_PROLOGUE(.x86shadowstack,,)
 {
 	MMU_PAGE_ALIGN
diff --git a/include/zephyr/arch/x86/ia32/structs.h b/include/zephyr/arch/x86/ia32/structs.h
index 114e6d7241941..1429e1cadc2e7 100644
--- a/include/zephyr/arch/x86/ia32/structs.h
+++ b/include/zephyr/arch/x86/ia32/structs.h
@@ -25,13 +25,13 @@ struct _cpu_arch {
 
 	struct k_thread *fpu_owner;
 #endif
-#if defined(CONFIG_X86_CET_SHADOW_STACK)
+#if defined(CONFIG_HW_SHADOW_STACK)
 	long *shstk_addr; /* Latest top of shadow stack */
 	long *shstk_base; /* Base of shadow stack */
 	size_t shstk_size;
 #endif
 #if defined(__cplusplus) && !defined(CONFIG_FPU_SHARING) && \
-		!defined(CONFIG_X86_CET_SHADOW_STACK)
+		!defined(CONFIG_HW_SHADOW_STACK)
 	/* Ensure this struct does not have a size of 0 which is not allowed in C++. */
 	uint8_t dummy;
 #endif
diff --git a/include/zephyr/arch/x86/ia32/thread.h b/include/zephyr/arch/x86/ia32/thread.h
index 7dbb61e621a38..b66ba31f835a9 100644
--- a/include/zephyr/arch/x86/ia32/thread.h
+++ b/include/zephyr/arch/x86/ia32/thread.h
@@ -237,7 +237,7 @@ struct _thread_arch {
 
 	tPreempFloatReg preempFloatReg; /* volatile float register storage */
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	long *shstk_addr;
 	long *shstk_base;
 	size_t shstk_size;
diff --git a/include/zephyr/arch/x86/intel64/linker.ld b/include/zephyr/arch/x86/intel64/linker.ld
index 04fb2a15bc885..f79a28507677d 100644
--- a/include/zephyr/arch/x86/intel64/linker.ld
+++ b/include/zephyr/arch/x86/intel64/linker.ld
@@ -189,7 +189,7 @@ SECTIONS
 #include <zephyr/linker/cplusplus-ram.ld>
 #include <zephyr/arch/x86/pagetables.ld>
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 SECTION_PROLOGUE(.x86shadowstack,,)
 {
     MMU_PAGE_ALIGN
diff --git a/include/zephyr/arch/x86/intel64/thread.h b/include/zephyr/arch/x86/intel64/thread.h
index b3c54d647bc62..809ad77222fbc 100644
--- a/include/zephyr/arch/x86/intel64/thread.h
+++ b/include/zephyr/arch/x86/intel64/thread.h
@@ -140,7 +140,7 @@ struct _thread_arch {
 	uint64_t cs;
 #endif
 
-#ifdef CONFIG_X86_CET_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK
 	uintptr_t *shstk_addr;
 	uintptr_t *shstk_base;
 	size_t shstk_size;

From 28947476d433cbd0718437b9bb72fab7cc740f9b Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Fri, 25 Apr 2025 13:46:38 -0700
Subject: [PATCH 14/21] kernel: Option to allow shadow stack to be reused

It seems that, at least on tests, it's common to call k_thread_create()
on a thread multiple times. This trips a check for the CET shadow stack
- namely, set a shadow stack on a thread which already has a shadow
stack.

This patch adds a Kconfig option to allow that, iff the base address and
size of the new shadow stack are the same as before. This will trigger a
reset of the shadow stack, so it can be reused.

It may be the case that this behaviour (reusing threads) is more common
than only for tests, in which case it could make sense to change the
default - in this patch, is only true if ZTEST.

Even if being enabled by default becomes the reality, it would still
make sense to keep this option - more conscious apps could avoid the
need for the shadow stack reset code altogether.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/core/cet.c            | 17 ++++++++++
 arch/x86/core/ia32/swap.S      | 60 ++++++++++++++++++++++++++++++++++
 arch/x86/core/intel64/locore.S | 39 ++++++++++++++++++++++
 kernel/Kconfig                 | 10 ++++++
 4 files changed, 126 insertions(+)

diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 3d2225f9c32df..010e3056961f0 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -25,6 +25,10 @@ LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
 #endif
 
 #ifdef CONFIG_HW_SHADOW_STACK
+#ifdef CONFIG_HW_SHADOW_STACK_ALLOW_REUSE
+extern void arch_shadow_stack_reset(k_tid_t thread);
+#endif /* CONFIG_HW_SHADOW_STACK_ALLOW_REUSE */
+
 int arch_thread_hw_shadow_stack_attach(k_tid_t thread,
 				       arch_thread_hw_shadow_stack_t *stack,
 				       size_t stack_size)
@@ -37,6 +41,19 @@ int arch_thread_hw_shadow_stack_attach(k_tid_t thread,
 
 	/* Or if the thread already has a shadow stack. */
 	if (thread->arch.shstk_addr != NULL) {
+#ifdef CONFIG_HW_SHADOW_STACK_ALLOW_REUSE
+		/* Allow reuse of the shadow stack if the base and size are the same */
+		if (thread->arch.shstk_base == stack &&
+		    thread->arch.shstk_size == stack_size) {
+			unsigned int key;
+
+			key = arch_irq_lock();
+			arch_shadow_stack_reset(thread);
+			arch_irq_unlock(key);
+
+			return 0;
+		}
+#endif
 		LOG_ERR("Shadow stack already set up for thread %p\n", thread);
 		return -EINVAL;
 	}
diff --git a/arch/x86/core/ia32/swap.S b/arch/x86/core/ia32/swap.S
index ff01e6b4a2ef1..4baa0070b9b8d 100644
--- a/arch/x86/core/ia32/swap.S
+++ b/arch/x86/core/ia32/swap.S
@@ -28,6 +28,10 @@
 	GTEXT(z_x86_thread_entry_wrapper)
 	GTEXT(_x86_user_thread_entry_wrapper)
 
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	GTEXT(arch_shadow_stack_reset)
+#endif
+
 	/* externs */
 #if !defined(CONFIG_X86_KPTI) && defined(CONFIG_X86_USERSPACE)
 	GTEXT(z_x86_swap_update_page_tables)
@@ -484,3 +488,59 @@ SECTION_FUNC(PINNED_TEXT, z_x86_thread_entry_wrapper)
 	movl	$0, (%esp)
 	jmp	*%edi
 #endif /* _THREAD_WRAPPER_REQUIRED */
+
+#ifdef CONFIG_HW_SHADOW_STACK_ALLOW_REUSE
+	/*
+	 * C function prototype:
+	 *
+	 * void arch_shadow_stack_reset(k_thread *thread);
+	 */
+SECTION_FUNC(PINNED_TEXT, arch_shadow_stack_reset)
+	movl 0x4(%esp), %eax /* *thread on EAX */
+	pushl %ebx
+
+	/* As we are touching other thread shadow stack memory, SoC may
+	 * need to do some house keeping.
+	 */
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	pushl %eax
+	soc_prepare_shadow_stack_switch
+	popl %eax
+#endif
+
+	movl _thread_offset_to_shstk_base(%eax), %ecx
+	movl _thread_offset_to_shstk_size(%eax), %edx
+
+	/* Set "shstk[-2]", z_thread_entry/z_x86_thread_entry_wrapper */
+	subl $8, %edx
+	addl %edx, %ecx
+	movl %ecx, %edx /* Save "&stsk[-2]" */
+#ifdef CONFIG_X86_DEBUG_INFO
+	movl $z_x86_thread_entry_wrapper, %ebx
+#else
+	movl $z_thread_entry, %ebx
+#endif
+	wrssd %ebx, (%ecx)
+
+	/* Set "shstk[-3]", token high bits, 0 */
+	xorl %ebx, %ebx
+	wrssd %ebx, -4(%ecx)
+
+	/* Set "shstk[-4]", token low bits */
+	wrssd %edx, -8(%ecx)
+
+	/* Set thread->shstk_addr to point to the token on the stack */
+	subl $8, %ecx
+	movl %ecx, _thread_offset_to_shstk_addr(%eax)
+
+#ifdef CONFIG_X86_CET_SOC_PREPARE_SHADOW_STACK_SWITCH
+	movl	$_kernel, %eax
+	movl _kernel_offset_to_current(%eax), %ecx
+	pushl %ecx
+	soc_prepare_shadow_stack_switch
+	popl %ecx
+#endif
+
+	popl %ebx
+	ret
+#endif
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index 439fea6619c9e..f45738dfabb5a 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -1024,6 +1024,45 @@ irq_exit_nested:
 	popq %rax
 	iretq
 
+#ifdef CONFIG_HW_SHADOW_STACK_ALLOW_REUSE
+	/*
+	 * C function prototype:
+	 *
+	 * void arch_shadow_stack_reset(k_thread *thread);
+	 */
+.globl arch_shadow_stack_reset
+arch_shadow_stack_reset:
+	movq _thread_offset_to_shstk_base(%rdi), %rcx
+	movq _thread_offset_to_shstk_size(%rdi), %rdx
+
+	/* Set "shstk[-2]", CS */
+	subq $16, %rdx
+	addq %rdx, %rcx
+	movq $X86_KERNEL_CS, %rsi
+	wrssq %rsi, (%rcx)
+
+	/* Set "shstk[-3]", z_thread_entry */
+	movq $z_thread_entry, %rsi
+	wrssq %rsi, -8(%rcx)
+
+	/* Set "shstk[-4]", ssp, "&shstk[-1]" */
+	movq %rcx, %rsi
+	addq $8, %rsi
+	wrssq %rsi, -16(%rcx)
+
+	/* Set "shstk[-5]", token, "&shstk[-4] | 1" */
+	movq %rcx, %rsi
+	subq $16, %rsi
+	orq $1, %rsi
+	wrssq %rsi, -24(%rcx)
+
+	/* Set thread->shstk_addr to point to the token on the stack */
+	subq $24, %rcx
+	movq %rcx, _thread_offset_to_shstk_addr(%rdi)
+
+	ret
+#endif
+
 #define IRQ(nr) vector_ ## nr: endbr64; pushq $(nr - IV_IRQS); jmp irq
 
 IRQ( 33); IRQ( 34); IRQ( 35); IRQ( 36); IRQ( 37); IRQ( 38); IRQ( 39)
diff --git a/kernel/Kconfig b/kernel/Kconfig
index c7ed58b7f1025..323ffbbd8e345 100644
--- a/kernel/Kconfig
+++ b/kernel/Kconfig
@@ -1045,6 +1045,16 @@ config HW_SHADOW_STACK
 	  stack. Indeed, threads need to manually attach a shadow stack to
 	  use this feature. See the documentation for more information.
 
+config HW_SHADOW_STACK_ALLOW_REUSE
+	bool "Allow reuse of the shadow stack"
+	depends on HW_SHADOW_STACK
+	default y if ZTEST
+	help
+	  This option allows the reuse of the shadow stack. This is meant to
+	  accommodate thread reuse, when the same thread is created again,
+	  such as done by the test suite. It is not meant to be used to share
+	  the shadow stack between threads.
+
 config KERNEL_HW_SHADOW_STACK_SIZE
 	int "Size of the hardware shadow stack"
 	depends on HW_SHADOW_STACK

From 31629dbb6986af8d563b6aa89b53a400319bf3e4 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Fri, 25 Apr 2025 14:38:05 -0700
Subject: [PATCH 15/21] tests/kernel/fatal/exception: Skip stack sentinel test
 with shadow stack

Test could trip shadow stack protections instead of normal stack
sentinel, thus requiring special handling for this case. Just avoid this
test instead, if CONFIG_HW_SHADOW_STACK=y.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 tests/kernel/fatal/exception/src/main.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/kernel/fatal/exception/src/main.c b/tests/kernel/fatal/exception/src/main.c
index b93aa0a60680f..048aabff103af 100644
--- a/tests/kernel/fatal/exception/src/main.c
+++ b/tests/kernel/fatal/exception/src/main.c
@@ -385,7 +385,7 @@ ZTEST(fatal_exception, test_fatal)
 
 #ifndef CONFIG_ARCH_POSIX
 
-#ifdef CONFIG_STACK_SENTINEL
+#if defined(CONFIG_STACK_SENTINEL) && !defined(CONFIG_HW_SHADOW_STACK)
 	TC_PRINT("test stack sentinel overflow - timer irq\n");
 	check_stack_overflow(stack_sentinel_timer, 0);
 

From 399b28ab40911d107136d318fd224bab856f1357 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Wed, 7 May 2025 13:18:09 -0700
Subject: [PATCH 16/21] cmake: New list for post build commands that should run
 early

New cmake list, `post_build_patch_elf_commands`, will be prepended to
`post_build_commands` one, effectively making these commands to run just
after the ELF is created. It's particularly useful for operations that
patch the ELF generated, before other representations of it (such as
.bin) are created.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 CMakeLists.txt | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc6f8266d6778..512d42cc668e0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2105,6 +2105,16 @@ if (CONFIG_BUILD_OUTPUT_VIF)
   include(${CMAKE_CURRENT_LIST_DIR}/cmake/vif.cmake)
 endif()
 
+get_property(post_build_patch_elf_commands
+  GLOBAL PROPERTY
+  post_build_patch_elf_commands
+  )
+
+list(PREPEND
+  post_build_commands
+  ${post_build_patch_elf_commands}
+  )
+
 get_property(extra_post_build_commands
   GLOBAL PROPERTY
   extra_post_build_commands

From d1b68c7b7f4a89d6d45e999f7ccac8029ad2af85 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Wed, 7 May 2025 13:20:33 -0700
Subject: [PATCH 17/21] kernel: Automatically set up HW shadow stack for thread
 stacks

This patch modifies thread stack macros (such as K_KERNEL_STACK_DECLARE
or K_KERNEL_STACK_ARRAY_DECLARE) to also create a HW shadow stack (when
CONFIG_HW_SHADOW_STACK=y), as well as define a pairing between the
thread stack (or thread stack array) and the shadow stack (or shadow
stack array).

This pairing, which currently is simply an array of pairs (stack,
shadow_stack) is searched during thread setup to find the corresponding
shadow stack and attach it to the thread. If linear search on this array
proves to be a performance issue, the actual structure can be revisited.

To define the size of the shadow stack for a given stack, the stack size
is used. A new Kconfig, CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE is used
to define how big the shadow stack is compared to the stack. Note that
this size is in *addition* to the stack size. To avoid some shadow
stacks becoming too small, CONFIG_HW_SHADOW_STACK_MIN_SIZE is used to
define a minimum size. Note that after this size is defined, platform
restrictions on the size of the shadow stack are still applied.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 include/zephyr/kernel/thread_stack.h          | 106 +++++++++++++++---
 .../common-rom/common-rom-kernel-devices.ld   |   5 +
 kernel/Kconfig                                |  25 +++++
 kernel/thread.c                               |  68 +++++++++++
 4 files changed, 190 insertions(+), 14 deletions(-)

diff --git a/include/zephyr/kernel/thread_stack.h b/include/zephyr/kernel/thread_stack.h
index 4f41de2580d06..9bdc0d9b188e0 100644
--- a/include/zephyr/kernel/thread_stack.h
+++ b/include/zephyr/kernel/thread_stack.h
@@ -112,6 +112,79 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @{
  */
 
+#ifdef CONFIG_HW_SHADOW_STACK
+#define k_thread_hw_shadow_stack_t arch_thread_hw_shadow_stack_t
+
+#define K_THREAD_HW_SHADOW_STACK_SIZE(size_) \
+	MAX(ROUND_UP((CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE * (size_) / 100), \
+		      CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT), \
+	    CONFIG_HW_SHADOW_STACK_MIN_SIZE)
+
+#define K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, size) \
+	ARCH_THREAD_HW_SHADOW_STACK_DECLARE(__ ## sym ## _shstk, size)
+
+#define K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DECLARE(__ ## sym ## _shstk_arr, \
+						  nmemb, size)
+
+struct _stack_to_hw_shadow_stack {
+	k_thread_stack_t *stack;
+	k_thread_hw_shadow_stack_t *shstk_addr;
+	size_t size;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_DEFINE(sym, size_) \
+	ARCH_THREAD_HW_SHADOW_STACK_DEFINE(__ ## sym ## _shstk, size_); \
+	static const STRUCT_SECTION_ITERABLE(_stack_to_hw_shadow_stack, \
+		sym ## _stack_to_shstk_attach) = { \
+		.stack = sym, \
+		.shstk_addr = __ ## sym ## _shstk, \
+		.size = size_, \
+	}
+
+struct _stack_to_hw_shadow_stack_arr {
+	uintptr_t stack_addr;
+	uintptr_t shstk_addr;
+	size_t stack_size;
+	size_t shstk_size;
+	size_t nmemb;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb_, size_) \
+	ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(__ ## sym ## _shstk_arr, nmemb_, \
+			K_THREAD_HW_SHADOW_STACK_SIZE(size_)); \
+	static const STRUCT_SECTION_ITERABLE(_stack_to_hw_shadow_stack_arr, \
+		sym ## _stack_to_shstk_attach) = { \
+		.stack_addr = (uintptr_t)sym, \
+		.stack_size = K_KERNEL_STACK_LEN(size_), \
+		.nmemb = nmemb_, \
+		.shstk_addr = (uintptr_t)__ ## sym ## _shstk_arr, \
+		.shstk_size = K_THREAD_HW_SHADOW_STACK_SIZE(size_), \
+	}
+
+#define k_thread_hw_shadow_stack_attach arch_thread_hw_shadow_stack_attach
+
+struct _thread_hw_shadow_stack_static {
+	struct k_thread *thread;
+	k_thread_hw_shadow_stack_t *shstk_addr;
+	size_t size;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_ATTACH(thread_, shstk_addr_, size_) \
+	static const STRUCT_SECTION_ITERABLE(_thread_hw_shadow_stack_static, \
+		thread ## _shstk_attach_static) = { \
+		.thread = thread_, \
+		.shstk_addr = shstk_addr_, \
+		.size = size_, \
+	}
+
+#else
+#define K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, size)
+#define K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, size)
+#define K_THREAD_HW_SHADOW_STACK_DEFINE(sym, size)
+#define K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size_)
+#endif
+
 /**
  * @brief Declare a reference to a thread stack
  *
@@ -122,6 +195,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_DECLARE(sym, size) \
+	K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[K_KERNEL_STACK_LEN(size)]
 
@@ -136,6 +210,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[nmemb][K_KERNEL_STACK_LEN(size)]
 
@@ -150,6 +225,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_PINNED_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[nmemb][K_KERNEL_STACK_LEN(size)]
 
@@ -212,7 +288,9 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 
 /**
  * @brief Define a toplevel kernel stack memory region in pinned section
@@ -228,10 +306,14 @@ static inline char *z_stack_ptr_align(char *ptr)
  */
 #if defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 #else
 #define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
 /**
@@ -244,7 +326,9 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
+
 
 /**
  * @brief Define a toplevel array of kernel stack memory regions in pinned section
@@ -261,10 +345,12 @@ static inline char *z_stack_ptr_align(char *ptr)
  */
 #if defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
 #else
 #define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
 /**
@@ -283,14 +369,6 @@ static inline char *z_stack_ptr_align(char *ptr)
 
 /** @} */
 
-#ifdef CONFIG_HW_SHADOW_STACK
-#define K_THREAD_HW_SHADOW_STACK_DEFINE ARCH_THREAD_HW_SHADOW_STACK_DEFINE
-
-#define k_thread_hw_shadow_stack_t arch_thread_hw_shadow_stack_t
-
-#define k_thread_hw_shadow_stack_attach arch_thread_hw_shadow_stack_attach
-#endif
-
 static inline char *K_KERNEL_STACK_BUFFER(k_thread_stack_t *sym)
 {
 	return (char *)sym + K_KERNEL_STACK_RESERVED;
diff --git a/include/zephyr/linker/common-rom/common-rom-kernel-devices.ld b/include/zephyr/linker/common-rom/common-rom-kernel-devices.ld
index 2bb7d2c509ce2..0cf3a031f2b43 100644
--- a/include/zephyr/linker/common-rom/common-rom-kernel-devices.ld
+++ b/include/zephyr/linker/common-rom/common-rom-kernel-devices.ld
@@ -102,4 +102,9 @@
 	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
 #endif /* !CONFIG_DEVICE_DEPS_DYNAMIC */
 
+#ifdef CONFIG_HW_SHADOW_STACK
+	ITERABLE_SECTION_ROM(_stack_to_hw_shadow_stack, Z_LINK_ITERABLE_SUBALIGN)
+	ITERABLE_SECTION_ROM(_stack_to_hw_shadow_stack_arr, Z_LINK_ITERABLE_SUBALIGN)
+#endif
+
 #include <device-api-sections.ld>
diff --git a/kernel/Kconfig b/kernel/Kconfig
index 323ffbbd8e345..68fdf774e9d21 100644
--- a/kernel/Kconfig
+++ b/kernel/Kconfig
@@ -1045,6 +1045,31 @@ config HW_SHADOW_STACK
 	  stack. Indeed, threads need to manually attach a shadow stack to
 	  use this feature. See the documentation for more information.
 
+config HW_SHADOW_STACK_PERCENTAGE_SIZE
+	int "Percentage of the stack size used to define shadow stack size"
+	depends on HW_SHADOW_STACK
+	default 30
+	range 0 100
+	help
+	  This option specifies the percentage of the stack to be used for
+	  shadow stack. The value is a percentage of the total stack size.
+	  The default value is 30%, which means that the shadow stack size
+	  will be 30% of the stack size, in *addition* to the stack size.
+	  used for shadow stack. Note that this size is constrained by the
+	  HW_SHADOW_STACK_MIN_SIZE config option.
+
+config HW_SHADOW_STACK_MIN_SIZE
+	int "Minimum size of the hardware shadow stack"
+	depends on HW_SHADOW_STACK
+	default 256
+	help
+	  This option specifies the minimum size of the hardware shadow stack,
+	  in bytes, so that threads that use minimal stack size can still
+	  have a sensible shadow stack size. The default value is 256 bytes,
+	  which means that the shadow stack size will be at least 256 bytes,
+	  even if the percentage defined via HW_SHADOW_STACK_PERCENTAGE_SIZE
+	  would define a smaller size.
+
 config HW_SHADOW_STACK_ALLOW_REUSE
 	bool "Allow reuse of the shadow stack"
 	depends on HW_SHADOW_STACK
diff --git a/kernel/thread.c b/kernel/thread.c
index f77d70ce494fb..bfcd9c87772fb 100644
--- a/kernel/thread.c
+++ b/kernel/thread.c
@@ -501,6 +501,70 @@ static char *setup_thread_stack(struct k_thread *new_thread,
 	return stack_ptr;
 }
 
+#ifdef CONFIG_HW_SHADOW_STACK
+static void setup_shadow_stack(struct k_thread *new_thread,
+			 k_thread_stack_t *stack)
+{
+	int ret = -ENOENT;
+
+	STRUCT_SECTION_FOREACH(_stack_to_hw_shadow_stack, stk_to_hw_shstk) {
+		if (stk_to_hw_shstk->stack == stack) {
+			ret = k_thread_hw_shadow_stack_attach(new_thread,
+							      stk_to_hw_shstk->shstk_addr,
+							      stk_to_hw_shstk->size);
+			if (ret != 0) {
+				LOG_ERR("Could not set thread %p shadow stack %p, got error %d",
+					new_thread, stk_to_hw_shstk->shstk_addr, ret);
+				k_panic();
+			}
+			break;
+		}
+	}
+
+	/* Check if the stack isn't in a stack array, and use the corresponding
+	 * shadow stack.
+	 */
+	if (ret != -ENOENT) {
+		return;
+	}
+
+	STRUCT_SECTION_FOREACH(_stack_to_hw_shadow_stack_arr, stk_to_hw_shstk) {
+		if ((uintptr_t)stack >= stk_to_hw_shstk->stack_addr &&
+		    (uintptr_t)stack < stk_to_hw_shstk->stack_addr +
+		    stk_to_hw_shstk->stack_size * stk_to_hw_shstk->nmemb) {
+			/* Now we have to guess which index of the stack array is being used */
+			uintptr_t stack_offset = (uintptr_t)stack - stk_to_hw_shstk->stack_addr;
+			uintptr_t stack_index = stack_offset / stk_to_hw_shstk->stack_size;
+			uintptr_t addr;
+
+			if (stack_index >= stk_to_hw_shstk->nmemb) {
+				LOG_ERR("Could not find shadow stack for thread %p, stack %p",
+					new_thread, stack);
+				k_panic();
+			}
+
+			addr = stk_to_hw_shstk->shstk_addr +
+				stk_to_hw_shstk->shstk_size * stack_index;
+			ret = k_thread_hw_shadow_stack_attach(new_thread,
+							      (arch_thread_hw_shadow_stack_t *)addr,
+							      stk_to_hw_shstk->shstk_size);
+			if (ret != 0) {
+				LOG_ERR("Could not set thread %p shadow stack 0x%lx, got error %d",
+					new_thread, stk_to_hw_shstk->shstk_addr, ret);
+				k_panic();
+			}
+			break;
+		}
+	}
+
+	if (ret == -ENOENT) {
+		LOG_ERR("Could not find shadow stack for thread %p, stack %p",
+			new_thread, stack);
+		k_panic();
+	}
+}
+#endif
+
 /*
  * The provided stack_size value is presumed to be either the result of
  * K_THREAD_STACK_SIZEOF(stack), or the size value passed to the instance
@@ -547,6 +611,10 @@ char *z_setup_new_thread(struct k_thread *new_thread,
 	z_init_thread_base(&new_thread->base, prio, _THREAD_SLEEPING, options);
 	stack_ptr = setup_thread_stack(new_thread, stack, stack_size);
 
+#ifdef CONFIG_HW_SHADOW_STACK
+	setup_shadow_stack(new_thread, stack);
+#endif
+
 #ifdef CONFIG_KERNEL_COHERENCE
 	/* Check that the thread object is safe, but that the stack is
 	 * still cached!

From d04b192b31cd507ce239efce5eb2ee686b241f01 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Wed, 7 May 2025 13:24:33 -0700
Subject: [PATCH 18/21] arch/x86: Support for automatic shadow stacks

   - No more need for special IRQ shadow stacks - just reuse the one
   created for z_interrupt_stacks;
   - Add the linker sections for the pairs of stack/shadow stack;
   - Support shadow stack arrays.

Last item was a bit challenging: shadow stacks need to be initialised
before use, and this is done statically for normal shadow stacks. To
initialise the shadow stacks in the array, one needs how many entries it
has. While a simple approach would use `LISTIFY` to them do the
initialization on all entries, that is not possible as many stack arrays
are created using expressions instead of literals, such as
`CONFIG_MP_MAX_NUM_CPUS - 1`, which won't work with `LISTIFY`.

Instead, this patch uses a script, `gen_static_shstk_array.py` that
gathers all needed information and patches the ELF to initialize the
stack arrays. Note that this needs to be done before any other operation
on the ELF file that creates new representations, such as the .bin
output.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 arch/x86/CMakeLists.txt                     |  28 +++
 arch/x86/Kconfig                            |   8 -
 arch/x86/core/cet.c                         |   1 +
 arch/x86/core/ia32/irq_manage.c             |  11 +-
 arch/x86/core/intel64/locore.S              |   7 +-
 arch/x86/gen_static_shstk_array.py          | 233 ++++++++++++++++++++
 arch/x86/include/intel64/kernel_arch_data.h |  18 +-
 include/zephyr/arch/x86/cet.h               |  19 +-
 include/zephyr/arch/x86/ia32/linker.ld      |  10 +
 include/zephyr/arch/x86/intel64/linker.ld   |  10 +
 10 files changed, 318 insertions(+), 27 deletions(-)
 create mode 100644 arch/x86/gen_static_shstk_array.py

diff --git a/arch/x86/CMakeLists.txt b/arch/x86/CMakeLists.txt
index 3bfb743f2421f..19621e60582f2 100644
--- a/arch/x86/CMakeLists.txt
+++ b/arch/x86/CMakeLists.txt
@@ -75,4 +75,32 @@ endif()
 
 if(CONFIG_X86_CET)
   zephyr_compile_options(-fcf-protection=full)
+
+  if(CONFIG_HW_SHADOW_STACK)
+    set(GEN_SHSTK_ARRAY ${ZEPHYR_BASE}/arch/x86/gen_static_shstk_array.py)
+
+    if(CONFIG_X86_64)
+      set(gen_shstk_array_arch_param --arch x86_64)
+
+      execute_process(
+        COMMAND
+        ${PYTHON_EXECUTABLE}
+        ${GEN_SHSTK_ARRAY}
+        --header-output ${PROJECT_BINARY_DIR}/include/generated/zephyr/x86_shstk.h
+        --config ${DOTCONFIG}
+        ${gen_shstk_array_arch_param}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      )
+    else()
+      set(gen_shstk_array_arch_param --arch x86)
+    endif()
+
+    set_property(GLOBAL APPEND PROPERTY post_build_patch_elf_commands
+      COMMAND
+      ${PYTHON_EXECUTABLE}
+      ${GEN_SHSTK_ARRAY}
+      --kernel $<TARGET_FILE:${ZEPHYR_FINAL_EXECUTABLE}>
+      ${gen_shstk_array_arch_param}
+    )
+  endif()
 endif()
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b8b8f08575b38..e014766f63449 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -515,14 +515,6 @@ config X86_CET_SHADOW_STACK_ALIGNMENT
 	help
 	  This option sets the alignment of the Shadow Stack.
 
-config X86_CET_IRQ_SHADOW_STACK_SIZE
-	int "IRQ Shadow Stack size"
-	default 4096
-	depends on HW_SHADOW_STACK
-	help
-	  This option sets the size of the shadow stack used for interrupt
-	  and exception handling.
-
 config X86_CET_VERIFY_KERNEL_SHADOW_STACK
 	bool "Verify kernel shadow stack"
 	depends on HW_SHADOW_STACK
diff --git a/arch/x86/core/cet.c b/arch/x86/core/cet.c
index 010e3056961f0..8636d7bb766d3 100644
--- a/arch/x86/core/cet.c
+++ b/arch/x86/core/cet.c
@@ -16,6 +16,7 @@
 #include <zephyr/logging/log_ctrl.h>
 #include <zephyr/logging/log.h>
 
+#include <kernel_arch_data.h>
 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/core/ia32/irq_manage.c b/arch/x86/core/ia32/irq_manage.c
index bb7f0b6397b47..6b3b554ac19bc 100644
--- a/arch/x86/core/ia32/irq_manage.c
+++ b/arch/x86/core/ia32/irq_manage.c
@@ -24,11 +24,6 @@
 #include <zephyr/arch/x86/ia32/segmentation.h>
 #include <kernel_arch_data.h>
 
-#ifdef CONFIG_HW_SHADOW_STACK
-X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack,
-			    CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);
-#endif
-
 #define TOKEN_OFFSET 4
 
 extern void z_SpuriousIntHandler(void *handler);
@@ -54,8 +49,10 @@ void arch_isr_direct_footer_swap(unsigned int key)
 #ifdef CONFIG_HW_SHADOW_STACK
 void z_x86_set_irq_shadow_stack(void)
 {
-	size_t stack_size = CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE;
-	arch_thread_hw_shadow_stack_t *stack = z_x86_irq_shadow_stack;
+	size_t stack_size = sizeof(__z_interrupt_stacks_shstk_arr);
+	arch_thread_hw_shadow_stack_t *stack;
+
+	stack = (arch_thread_hw_shadow_stack_t *)__z_interrupt_stacks_shstk_arr;
 
 	_kernel.cpus[0].arch.shstk_addr = stack +
 		(stack_size - TOKEN_OFFSET * sizeof(*stack)) / sizeof(*stack);
diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S
index f45738dfabb5a..e2076814141a9 100644
--- a/arch/x86/core/intel64/locore.S
+++ b/arch/x86/core/intel64/locore.S
@@ -14,6 +14,9 @@
 #include <zephyr/drivers/interrupt_controller/loapic.h>
 #include <zephyr/arch/cpu.h>
 #include <zephyr/kernel/mm.h>
+#ifdef CONFIG_HW_SHADOW_STACK
+#include <x86_shstk.h>
+#endif
 
 /*
  * Definitions/macros for enabling paging
@@ -823,7 +826,7 @@ irq:
 #ifdef CONFIG_HW_SHADOW_STACK
 	pushq %rsi
 	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rsi
-	subq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rsi)
+	subq $X86_CET_IRQ_SHADOW_SUBSTACK_SIZE, (%rsi)
 	popq %rsi
 #endif
 	cmpl $CONFIG_ISR_DEPTH, ___cpu_t_nested_OFFSET(%rsi)
@@ -981,7 +984,7 @@ irq_dispatch:
 	addq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
 #ifdef CONFIG_HW_SHADOW_STACK
 	movq %gs:(__x86_tss64_t_shstk_addr_OFFSET), %rax
-	addq $CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE, (%rax)
+	addq $X86_CET_IRQ_SHADOW_SUBSTACK_SIZE, (%rax)
 #endif
 	decl ___cpu_t_nested_OFFSET(%rsi)
 	jnz irq_exit_nested
diff --git a/arch/x86/gen_static_shstk_array.py b/arch/x86/gen_static_shstk_array.py
new file mode 100644
index 0000000000000..29183da50c85a
--- /dev/null
+++ b/arch/x86/gen_static_shstk_array.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2025 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import struct
+
+from elftools.elf.elffile import ELFFile
+from elftools.elf.sections import SymbolTableSection
+
+
+def parse_args():
+    global args
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        allow_abbrev=False,
+    )
+
+    parser.add_argument("-k", "--kernel", required=False, help="Zephyr kernel image")
+    parser.add_argument("-o", "--header-output", required=False, help="Header output file")
+    parser.add_argument("-c", "--config", required=False, help="Configuration file (.config)")
+    parser.add_argument(
+        "-a",
+        "--arch",
+        required=False,
+        help="Architecture to generate shadow stack for",
+        choices=["x86", "x86_64"],
+        default="x86",
+    )
+    args = parser.parse_args()
+
+
+def get_symbols(obj):
+    for section in obj.iter_sections():
+        if isinstance(section, SymbolTableSection):
+            return {sym.name: sym for sym in section.iter_symbols()}
+
+    raise LookupError("Could not find symbol table")
+
+
+shstk_irq_top_fmt = "<Q"
+
+
+def generate_initialized_irq_array64(sym, section_data, section_addr, isr_depth):
+    offset = sym["st_value"] - section_addr
+    # First four bytes have the number of members of the array
+    nmemb = int.from_bytes(section_data[offset : offset + 4], "little") * isr_depth
+    stack_size = (int)(sym["st_size"] / nmemb)
+
+    # Top of shadow stack is on the form:
+    # [-1] = shadow stack supervisor token
+
+    output = bytearray(sym["st_size"])
+    for i in range(nmemb):
+        token = sym["st_value"] + stack_size * (i + 1) - 8
+
+        struct.pack_into(shstk_irq_top_fmt, output, stack_size * (i + 1) - 8, token)
+
+    return output
+
+
+shstk_top_fmt = "<QQQQ"
+
+
+def generate_initialized_array64(sym, section_data, section_addr, thread_entry):
+    offset = sym["st_value"] - section_addr
+    # First four bytes have the number of members of the array
+    nmemb = int.from_bytes(section_data[offset : offset + 4], "little")
+    if nmemb == 0:
+        # CONFIG_DYNAMIC_THREAD_POOL_SIZE can be zero - in which case isn't
+        # used. To allow the static initialization, we make the corresponding shadow stack
+        # have one item. Here, we recognize it by having 0 members. But we already
+        # wasted the space for this. Sigh...
+        return bytearray(sym["st_size"])
+    stack_size = (int)(sym["st_size"] / nmemb)
+
+    # Top of shadow stack is on the form:
+    # [-5] = shadow stack token, pointing to [-4]
+    # [-4] = previous SSP, pointing to [-1]
+    # [-3] = z_thread_entry
+    # [-2] = X86_KERNEL_CS
+
+    output = bytearray(sym["st_size"])
+    for i in range(nmemb):
+        end = sym["st_value"] + stack_size * (i + 1)
+        token = end - 8 * 4 + 1
+        prev_ssp = end - 8
+        cs = 0x18  # X86_KERNEL_CS
+
+        struct.pack_into(
+            shstk_top_fmt,
+            output,
+            stack_size * (i + 1) - 8 * 5,
+            token,
+            prev_ssp,
+            thread_entry["st_value"],
+            cs,
+        )
+
+    return output
+
+
+shstk_top_fmt32 = "<QI"
+
+
+def generate_initialized_array32(sym, section_data, section_addr, thread_entry):
+    offset = sym["st_value"] - section_addr
+    # First four bytes have the number of members of the array
+    nmemb = int.from_bytes(section_data[offset : offset + 4], "little")
+    if nmemb == 0:
+        # See comment on generate_initialized_array64
+        return bytearray(sym["st_size"])
+    stack_size = (int)(sym["st_size"] / nmemb)
+
+    # Top of shadow stack is on the form:
+    # [-4] = shadow stack token, pointing to [-2]
+    # [-3] = 0 - high order bits of token
+    # [-2] = z_thread_entry/z_x86_thread_entry_wrapper
+
+    output = bytearray(sym["st_size"])
+    for i in range(nmemb):
+        end = sym["st_value"] + stack_size * (i + 1)
+        token = end - 8
+
+        struct.pack_into(
+            shstk_top_fmt32,
+            output,
+            stack_size * (i + 1) - 4 * 4,
+            token,
+            thread_entry["st_value"],
+        )
+
+    return output
+
+
+def generate_initialized_array(sym, section_data, section_addr, thread_entry):
+    if args.arch == "x86":
+        return generate_initialized_array32(sym, section_data, section_addr, thread_entry)
+
+    return generate_initialized_array64(sym, section_data, section_addr, thread_entry)
+
+
+def patch_elf():
+    with open(args.kernel, "r+b") as elf_fp:
+        kernel = ELFFile(elf_fp)
+        section = kernel.get_section_by_name(".x86shadowstack.arr")
+        syms = get_symbols(kernel)
+        thread_entry = syms["z_thread_entry"]
+        if args.arch == "x86" and syms["CONFIG_X86_DEBUG_INFO"].entry.st_value:
+            thread_entry = syms["z_x86_thread_entry_wrapper"]
+
+        updated_section = bytearray()
+        shstk_arr_syms = [
+            sym[1]
+            for sym in syms.items()
+            if sym[0].startswith("__") and sym[0].endswith("_shstk_arr")
+        ]
+        shstk_arr_syms.sort(key=lambda x: x["st_value"])
+        section_data = section.data()
+
+        for sym in shstk_arr_syms:
+            if sym.name == "__z_interrupt_stacks_shstk_arr" and args.arch == "x86_64":
+                isr_depth = syms["CONFIG_ISR_DEPTH"].entry.st_value
+                out = generate_initialized_irq_array64(
+                    sym, section_data, section["sh_addr"], isr_depth
+                )
+            else:
+                out = generate_initialized_array(
+                    sym, section_data, section["sh_addr"], thread_entry
+                )
+
+            updated_section += out
+
+        elf_fp.seek(section["sh_offset"])
+        elf_fp.write(updated_section)
+
+
+def generate_header():
+    if args.config is None:
+        raise ValueError("Configuration file is required to generate header")
+
+    isr_depth = stack_size = alignment = hw_stack_percentage = hw_stack_min_size = None
+
+    with open(args.config) as config_fp:
+        config_lines = config_fp.readlines()
+
+    for line in config_lines:
+        if line.startswith("CONFIG_ISR_DEPTH="):
+            isr_depth = int(line.split("=")[1].strip())
+        if line.startswith("CONFIG_ISR_STACK_SIZE="):
+            stack_size = int(line.split("=")[1].strip())
+        if line.startswith("CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT="):
+            alignment = int(line.split("=")[1].strip())
+        if line.startswith("CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE="):
+            hw_stack_percentage = int(line.split("=")[1].strip())
+        if line.startswith("CONFIG_HW_SHADOW_STACK_MIN_SIZE="):
+            hw_stack_min_size = int(line.split("=")[1].strip())
+
+    if isr_depth is None:
+        raise ValueError("Missing CONFIG_ISR_DEPTH in configuration file")
+    if stack_size is None:
+        raise ValueError("Missing CONFIG_ISR_STACK_SIZE in configuration file")
+    if alignment is None:
+        raise ValueError("Missing CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT in configuration file")
+    if hw_stack_percentage is None:
+        raise ValueError("Missing CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE in configuration file")
+    if hw_stack_min_size is None:
+        raise ValueError("Missing CONFIG_HW_SHADOW_STACK_MIN_SIZE in configuration file")
+
+    stack_size = int(stack_size * (hw_stack_percentage / 100))
+    stack_size = int((stack_size + alignment - 1) / alignment) * alignment
+    stack_size = max(stack_size, hw_stack_min_size)
+    stack_size = int(stack_size / isr_depth)
+
+    with open(args.header_output, "w") as header_fp:
+        header_fp.write(f"#define X86_CET_IRQ_SHADOW_SUBSTACK_SIZE {stack_size}\n")
+
+
+def main():
+    parse_args()
+
+    if args.kernel is not None:
+        patch_elf()
+
+    if args.header_output is not None:
+        generate_header()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/arch/x86/include/intel64/kernel_arch_data.h b/arch/x86/include/intel64/kernel_arch_data.h
index da24fc65a791f..a1e4c25427200 100644
--- a/arch/x86/include/intel64/kernel_arch_data.h
+++ b/arch/x86/include/intel64/kernel_arch_data.h
@@ -44,7 +44,6 @@ struct x86_interrupt_ssp_table {
 };
 
 extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
-
 #endif /* _ASMLANGUAGE */
 
 #ifdef CONFIG_X86_KPTI
@@ -91,6 +90,9 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 			(uintptr_t)name + size * (i + 1) - 1 *					\
 			sizeof(arch_thread_hw_shadow_stack_t)
 
+#define X86_EXCEPTION_SHADOW_STACK_SIZE \
+	K_THREAD_HW_SHADOW_STACK_SIZE(CONFIG_X86_EXCEPTION_STACK_SIZE)
+
 #define X86_IRQ_SHADOW_STACK_DEFINE(name, size)							\
 	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack)			\
 	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT)					\
@@ -100,24 +102,24 @@ extern uint8_t x86_cpu_loapics[];	/* CPU logical ID -> local APIC ID */
 		}
 
 #define X86_INTERRUPT_SHADOW_STACK_DEFINE(n)							\
-	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_irq_shadow_stack##n,					\
-				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);			\
 	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_nmi_shadow_stack##n,					\
-				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE);			\
+				     X86_EXCEPTION_SHADOW_STACK_SIZE);				\
 	X86_IRQ_SHADOW_STACK_DEFINE(z_x86_exception_shadow_stack##n,				\
-				     CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE)
+				     X86_EXCEPTION_SHADOW_STACK_SIZE)
 
-#define SHSTK_LAST_ENTRY (CONFIG_X86_CET_IRQ_SHADOW_STACK_SIZE *				\
+#define SHSTK_LAST_ENTRY (X86_EXCEPTION_SHADOW_STACK_SIZE *					\
 			  CONFIG_ISR_DEPTH / sizeof(arch_thread_hw_shadow_stack_t) - 1)
 
+#define IRQ_SHSTK_LAST_ENTRY sizeof(__z_interrupt_stacks_shstk_arr[0]) /			\
+	sizeof(arch_thread_hw_shadow_stack_t) - 1
+
 #define X86_INTERRUPT_SSP_TABLE_INIT(n, _)							\
 	{											\
-		.ist1 = (uintptr_t)&z_x86_irq_shadow_stack##n[SHSTK_LAST_ENTRY],		\
+		.ist1 = (uintptr_t)&__z_interrupt_stacks_shstk_arr[n][IRQ_SHSTK_LAST_ENTRY],	\
 		.ist6 = (uintptr_t)&z_x86_nmi_shadow_stack##n[SHSTK_LAST_ENTRY],		\
 		.ist7 = (uintptr_t)&z_x86_exception_shadow_stack##n[SHSTK_LAST_ENTRY],	\
 	}
 
-
 #define STACK_ARRAY_IDX(n, _) n
 
 #define DEFINE_STACK_ARRAY_IDX\
diff --git a/include/zephyr/arch/x86/cet.h b/include/zephyr/arch/x86/cet.h
index fd366c4286548..0839d42ceeaa0 100644
--- a/include/zephyr/arch/x86/cet.h
+++ b/include/zephyr/arch/x86/cet.h
@@ -17,6 +17,21 @@ extern FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry,
 
 typedef uintptr_t arch_thread_hw_shadow_stack_t;
 
+#define ARCH_THREAD_HW_SHADOW_STACK_DECLARE(sym, size) \
+	extern arch_thread_hw_shadow_stack_t sym[size / sizeof(arch_thread_hw_shadow_stack_t)]
+
+#define ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	extern arch_thread_hw_shadow_stack_t \
+		sym[nmemb][size / sizeof(arch_thread_hw_shadow_stack_t)]
+
+#define ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(name, nmemb, size) \
+	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack.arr_ ##name) \
+	__aligned(CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT) \
+	name[MAX(nmemb, 1)][size / sizeof(arch_thread_hw_shadow_stack_t)] = \
+		{ \
+			[0][0] = nmemb, \
+		}
+
 #ifdef CONFIG_X86_64
 #define ARCH_THREAD_HW_SHADOW_STACK_DEFINE(name, size) \
 	arch_thread_hw_shadow_stack_t Z_GENERIC_SECTION(.x86shadowstack) \
@@ -30,7 +45,8 @@ typedef uintptr_t arch_thread_hw_shadow_stack_t;
 		   (uintptr_t)z_thread_entry, \
 		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 2] = \
 		   (uintptr_t)X86_KERNEL_CS }
-#else
+
+#else /* CONFIG_X86_64 */
 
 #ifdef CONFIG_X86_DEBUG_INFO
 extern void z_x86_thread_entry_wrapper(k_thread_entry_t entry,
@@ -50,7 +66,6 @@ extern void z_x86_thread_entry_wrapper(k_thread_entry_t entry,
 		  [size / sizeof(arch_thread_hw_shadow_stack_t) - 2] = \
 		   (uintptr_t)___x86_entry_point, \
 		}
-
 #endif /* CONFIG_X86_64 */
 
 int arch_thread_hw_shadow_stack_attach(struct k_thread *thread,
diff --git a/include/zephyr/arch/x86/ia32/linker.ld b/include/zephyr/arch/x86/ia32/linker.ld
index 6bdf481e1968e..22e86a0716b05 100644
--- a/include/zephyr/arch/x86/ia32/linker.ld
+++ b/include/zephyr/arch/x86/ia32/linker.ld
@@ -482,7 +482,17 @@ SECTION_PROLOGUE(.x86shadowstack,,)
 	MMU_PAGE_ALIGN
 
 	__x86shadowstack_end = .;
+} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+SECTION_PROLOGUE(.x86shadowstack.arr,,)
+{
+	MMU_PAGE_ALIGN
 
+	*(.x86shadowstack.arr*)
+
+	MMU_PAGE_ALIGN
+
+	__x86shadowstack_end = .;
 } GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
 #endif
 
diff --git a/include/zephyr/arch/x86/intel64/linker.ld b/include/zephyr/arch/x86/intel64/linker.ld
index f79a28507677d..b20f40258b12b 100644
--- a/include/zephyr/arch/x86/intel64/linker.ld
+++ b/include/zephyr/arch/x86/intel64/linker.ld
@@ -200,6 +200,16 @@ SECTION_PROLOGUE(.x86shadowstack,,)
 
     MMU_PAGE_ALIGN
 
+} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+SECTION_PROLOGUE(.x86shadowstack.arr,,)
+{
+    MMU_PAGE_ALIGN
+
+    *(.x86shadowstack.arr*)
+
+    MMU_PAGE_ALIGN
+
     __x86shadowstack_end = .;
 
 } GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)

From 9a96a789464823987584bb15d94773c787bb3c8c Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Mon, 5 May 2025 18:02:17 -0700
Subject: [PATCH 19/21] tests/kernel/fifo/fifo_api: Don't reuse name for
 different thread stacks

A limitation on current HW shadow stack implementation for x86 is that
more than one thread stack can't share the same variable name, even if
they are static. To avoid skipping useful tests, for now, while this
limitation holds, change the name of the stack variable.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 tests/kernel/fifo/fifo_api/src/test_fifo_cancel.c   | 4 ++--
 tests/kernel/fifo/fifo_api/src/test_fifo_contexts.c | 4 ++--
 tests/kernel/fifo/fifo_api/src/test_fifo_loop.c     | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/kernel/fifo/fifo_api/src/test_fifo_cancel.c b/tests/kernel/fifo/fifo_api/src/test_fifo_cancel.c
index daa1b46f89a10..950b0084988da 100644
--- a/tests/kernel/fifo/fifo_api/src/test_fifo_cancel.c
+++ b/tests/kernel/fifo/fifo_api/src/test_fifo_cancel.c
@@ -13,7 +13,7 @@ K_FIFO_DEFINE(kfifo_c);
 
 struct k_fifo fifo_c;
 
-static K_THREAD_STACK_DEFINE(tstack, STACK_SIZE);
+static K_THREAD_STACK_DEFINE(tstack_cancel, STACK_SIZE);
 static struct k_thread thread;
 
 static void t_cancel_wait_entry(void *p1, void *p2, void *p3)
@@ -24,7 +24,7 @@ static void t_cancel_wait_entry(void *p1, void *p2, void *p3)
 
 static void tfifo_thread_thread(struct k_fifo *pfifo)
 {
-	k_tid_t tid = k_thread_create(&thread, tstack, STACK_SIZE,
+	k_tid_t tid = k_thread_create(&thread, tstack_cancel, STACK_SIZE,
 				      t_cancel_wait_entry, pfifo, NULL, NULL,
 				      K_PRIO_PREEMPT(0), 0, K_NO_WAIT);
 	uint32_t start_t = k_uptime_get_32();
diff --git a/tests/kernel/fifo/fifo_api/src/test_fifo_contexts.c b/tests/kernel/fifo/fifo_api/src/test_fifo_contexts.c
index a50f665b25db6..32d81e78750f8 100644
--- a/tests/kernel/fifo/fifo_api/src/test_fifo_contexts.c
+++ b/tests/kernel/fifo/fifo_api/src/test_fifo_contexts.c
@@ -16,7 +16,7 @@ static fdata_t data[LIST_LEN];
 static fdata_t data_l[LIST_LEN];
 static fdata_t data_sl[LIST_LEN];
 
-static K_THREAD_STACK_DEFINE(tstack, STACK_SIZE);
+static K_THREAD_STACK_DEFINE(tstack_contexts, STACK_SIZE);
 static struct k_thread tdata;
 static struct k_sem end_sema;
 
@@ -88,7 +88,7 @@ static void tfifo_thread_thread(struct k_fifo *pfifo)
 {
 	k_sem_init(&end_sema, 0, 1);
 	/**TESTPOINT: thread-thread data passing via fifo*/
-	k_tid_t tid = k_thread_create(&tdata, tstack, STACK_SIZE,
+	k_tid_t tid = k_thread_create(&tdata, tstack_contexts, STACK_SIZE,
 				      tThread_entry, pfifo, NULL, NULL,
 				      K_PRIO_PREEMPT(0), 0, K_NO_WAIT);
 	tfifo_put(pfifo);
diff --git a/tests/kernel/fifo/fifo_api/src/test_fifo_loop.c b/tests/kernel/fifo/fifo_api/src/test_fifo_loop.c
index 9e57dff7fec81..084ffd66e540b 100644
--- a/tests/kernel/fifo/fifo_api/src/test_fifo_loop.c
+++ b/tests/kernel/fifo/fifo_api/src/test_fifo_loop.c
@@ -12,7 +12,7 @@
 
 static fdata_t data[LIST_LEN];
 static struct k_fifo fifo;
-static K_THREAD_STACK_DEFINE(tstack, STACK_SIZE);
+static K_THREAD_STACK_DEFINE(tstack_loop, STACK_SIZE);
 static struct k_thread tdata;
 static struct k_sem end_sema;
 
@@ -60,7 +60,7 @@ static void tfifo_read_write(struct k_fifo *pfifo)
 {
 	k_sem_init(&end_sema, 0, 1);
 	/**TESTPOINT: thread-isr-thread data passing via fifo*/
-	k_tid_t tid = k_thread_create(&tdata, tstack, STACK_SIZE,
+	k_tid_t tid = k_thread_create(&tdata, tstack_loop, STACK_SIZE,
 				      tThread_entry, pfifo, NULL, NULL,
 				      K_PRIO_PREEMPT(0), 0, K_NO_WAIT);
 

From 064987ff256163f7dcb7f222d01f75517a161219 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Tue, 6 May 2025 09:21:20 -0700
Subject: [PATCH 20/21] tests/kernel/lifo/lifo_api: Don't reuse name for
 different thread stacks

A limitation on current HW shadow stack implementation for x86 is that
more than one thread stack can't share the same variable name, even if
they are static. To avoid skipping useful tests, for now, while this
limitation holds, change the name of the stack variable.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 tests/kernel/lifo/lifo_api/src/test_lifo_contexts.c | 4 ++--
 tests/kernel/lifo/lifo_api/src/test_lifo_loop.c     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/kernel/lifo/lifo_api/src/test_lifo_contexts.c b/tests/kernel/lifo/lifo_api/src/test_lifo_contexts.c
index 1f32f151cc6ac..3e4ed72dd45b8 100644
--- a/tests/kernel/lifo/lifo_api/src/test_lifo_contexts.c
+++ b/tests/kernel/lifo/lifo_api/src/test_lifo_contexts.c
@@ -14,7 +14,7 @@ K_LIFO_DEFINE(klifo);
 struct k_lifo lifo;
 static ldata_t data[LIST_LEN];
 
-static K_THREAD_STACK_DEFINE(tstack, STACK_SIZE);
+static K_THREAD_STACK_DEFINE(tstack_contexts, STACK_SIZE);
 static struct k_thread tdata;
 static struct k_sem end_sema;
 
@@ -59,7 +59,7 @@ static void tlifo_thread_thread(struct k_lifo *plifo)
 {
 	k_sem_init(&end_sema, 0, 1);
 	/**TESTPOINT: thread-thread data passing via lifo*/
-	k_tid_t tid = k_thread_create(&tdata, tstack, STACK_SIZE,
+	k_tid_t tid = k_thread_create(&tdata, tstack_contexts, STACK_SIZE,
 		tThread_entry, plifo, NULL, NULL,
 		K_PRIO_PREEMPT(0), 0, K_NO_WAIT);
 	tlifo_put(plifo);
diff --git a/tests/kernel/lifo/lifo_api/src/test_lifo_loop.c b/tests/kernel/lifo/lifo_api/src/test_lifo_loop.c
index c7e7310932c46..0f20fb3e61d89 100644
--- a/tests/kernel/lifo/lifo_api/src/test_lifo_loop.c
+++ b/tests/kernel/lifo/lifo_api/src/test_lifo_loop.c
@@ -12,7 +12,7 @@
 
 static ldata_t data[LIST_LEN];
 static struct k_lifo lifo;
-static K_THREAD_STACK_DEFINE(tstack, STACK_SIZE);
+static K_THREAD_STACK_DEFINE(tstack_loop, STACK_SIZE);
 static struct k_thread tdata;
 static struct k_sem end_sema;
 
@@ -60,7 +60,7 @@ static void tlifo_read_write(struct k_lifo *plifo)
 {
 	k_sem_init(&end_sema, 0, 1);
 	/**TESTPOINT: thread-isr-thread data passing via lifo*/
-	k_tid_t tid = k_thread_create(&tdata, tstack, STACK_SIZE,
+	k_tid_t tid = k_thread_create(&tdata, tstack_loop, STACK_SIZE,
 		tThread_entry, plifo, NULL, NULL,
 		K_PRIO_PREEMPT(0), 0, K_NO_WAIT);
 

From 3fc476dc63753cb03ede184c04c4f0f377f3dac5 Mon Sep 17 00:00:00 2001
From: Ederson de Souza <ederson.desouza@intel.com>
Date: Wed, 7 May 2025 14:00:07 -0700
Subject: [PATCH 21/21] tests/arch/x86: Add tests for shadow stack

Ensure they behave as expected.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
---
 tests/arch/x86/cet/prj.conf      |   1 +
 tests/arch/x86/cet/src/main.c    | 143 +++++++++++++++++++++++++++++--
 tests/arch/x86/cet/testcase.yaml |  11 +++
 3 files changed, 146 insertions(+), 9 deletions(-)

diff --git a/tests/arch/x86/cet/prj.conf b/tests/arch/x86/cet/prj.conf
index 54be14f4093c9..1f3f9c819682d 100644
--- a/tests/arch/x86/cet/prj.conf
+++ b/tests/arch/x86/cet/prj.conf
@@ -1,2 +1,3 @@
 CONFIG_ZTEST=y
 CONFIG_X86_CET=y
+CONFIG_IRQ_OFFLOAD=y
diff --git a/tests/arch/x86/cet/src/main.c b/tests/arch/x86/cet/src/main.c
index 4c848528d57aa..76ffb3ffd0d2a 100644
--- a/tests/arch/x86/cet/src/main.c
+++ b/tests/arch/x86/cet/src/main.c
@@ -5,32 +5,38 @@
  */
 
 #include <zephyr/ztest.h>
-#include <zephyr/arch/exception.h>
+#include <zephyr/irq_offload.h>
+
+#include <zephyr/kernel/thread_stack.h>
 
 #define IV_CTRL_PROTECTION_EXCEPTION 21
 
+#define CTRL_PROTECTION_ERRORCODE_NEAR_RET 1
 #define CTRL_PROTECTION_ERRORCODE_ENDBRANCH 3
 
-extern int should_work(int a);
-extern int should_not_work(int a);
+#define STACKSIZE 1024
+#define THREAD_PRIORITY 5
+
+K_SEM_DEFINE(error_handler_sem, 0, 1);
 
-static bool expect_fault;
-static int expect_code;
+volatile bool expect_fault;
+volatile int expect_code;
+volatile int expect_reason;
 
 void k_sys_fatal_error_handler(unsigned int reason, const struct arch_esf *pEsf)
 {
 	if (expect_fault) {
 #ifdef CONFIG_X86_64
-		zassert_equal(pEsf->vector, IV_CTRL_PROTECTION_EXCEPTION,
-					  "unexpected exception");
+		zassert_equal(pEsf->vector, expect_reason, "unexpected exception");
 		zassert_equal(pEsf->code, expect_code, "unexpected error code");
 #else
-		zassert_equal(z_x86_exception_vector, IV_CTRL_PROTECTION_EXCEPTION,
-					  "unexpected exception");
+		zassert_equal(z_x86_exception_vector, expect_reason, "unexpected exception");
 		zassert_equal(pEsf->errorCode, expect_code, "unexpected error code");
 #endif
 		printk("fatal error expected as part of test case\n");
 		expect_fault = false;
+
+		k_sem_give(&error_handler_sem);
 	} else {
 		printk("fatal error was unexpected, aborting\n");
 		TC_END_REPORT(TC_FAIL);
@@ -38,6 +44,123 @@ void k_sys_fatal_error_handler(unsigned int reason, const struct arch_esf *pEsf)
 	}
 }
 
+#ifdef CONFIG_HW_SHADOW_STACK
+void thread_a_entry(void *p1, void *p2, void *p3);
+K_SEM_DEFINE(thread_a_sem, 0, 1);
+K_THREAD_DEFINE(thread_a, STACKSIZE, thread_a_entry, NULL, NULL, NULL,
+		THREAD_PRIORITY, 0, -1);
+
+void thread_b_entry(void *p1, void *p2, void *p3);
+K_SEM_DEFINE(thread_b_sem, 0, 1);
+K_SEM_DEFINE(thread_b_irq_sem, 0, 1);
+K_THREAD_DEFINE(thread_b, STACKSIZE, thread_b_entry, NULL, NULL, NULL,
+		THREAD_PRIORITY, 0, -1);
+
+static bool is_shstk_enabled(void)
+{
+	long cur;
+
+	cur = z_x86_msr_read(X86_S_CET_MSR);
+	return (cur & X86_S_CET_MSR_SHSTK_EN) == X86_S_CET_MSR_SHSTK_EN;
+}
+
+void thread_c_entry(void *p1, void *p2, void *p3)
+{
+	zassert_true(is_shstk_enabled(), "shadow stack not enabled on static thread");
+}
+
+K_THREAD_DEFINE(thread_c, STACKSIZE, thread_c_entry, NULL, NULL, NULL,
+		THREAD_PRIORITY, 0, 0);
+
+void __attribute__((optimize("O0"))) foo(void)
+{
+	printk("foo called\n");
+}
+
+void __attribute__((optimize("O0"))) fail(void)
+{
+	long a[] = {0};
+
+	printk("should fail after this\n");
+
+	*(a + 2) = (long)&foo;
+}
+
+struct k_work work;
+
+void work_handler(struct k_work *wrk)
+{
+	printk("work handler\n");
+
+	zassert_true(is_shstk_enabled(), "shadow stack not enabled");
+}
+
+ZTEST(cet, test_shstk_work_q)
+{
+	k_work_init(&work, work_handler);
+	k_work_submit(&work);
+}
+
+void intr_handler(const void *p)
+{
+	printk("interrupt handler\n");
+
+	if (p != NULL) {
+		/* Test one nested level. It should just work. */
+		printk("trying interrupt handler\n");
+		irq_offload(intr_handler, NULL);
+
+		k_sem_give((struct k_sem *)p);
+	} else {
+		printk("interrupt handler nested\n");
+	}
+}
+
+void thread_b_entry(void *p1, void *p2, void *p3)
+{
+	k_sem_take(&thread_b_sem, K_FOREVER);
+
+	irq_offload(intr_handler, &thread_b_irq_sem);
+
+	k_sem_take(&thread_b_irq_sem, K_FOREVER);
+}
+
+ZTEST(cet, test_shstk_irq)
+{
+	k_thread_start(thread_b);
+
+	k_sem_give(&thread_b_sem);
+
+	k_thread_join(thread_b, K_FOREVER);
+}
+
+void thread_a_entry(void *p1, void *p2, void *p3)
+{
+	k_sem_take(&thread_a_sem, K_FOREVER);
+
+	fail();
+
+	zassert_unreachable("should not reach here");
+}
+
+ZTEST(cet, test_shstk)
+{
+	k_thread_start(thread_a);
+
+	expect_fault = true;
+	expect_code = CTRL_PROTECTION_ERRORCODE_NEAR_RET;
+	expect_reason = IV_CTRL_PROTECTION_EXCEPTION;
+	k_sem_give(&thread_a_sem);
+
+	k_sem_take(&error_handler_sem, K_FOREVER);
+	k_thread_abort(thread_a);
+}
+#endif /* CONFIG_HW_SHADOW_STACK */
+
+#ifdef CONFIG_X86_CET_IBT
+extern int should_work(int a);
+extern int should_not_work(int a);
+
 /* Round trip to trick optimisations and ensure the calls are indirect */
 int do_call(int (*func)(int), int a)
 {
@@ -50,8 +173,10 @@ ZTEST(cet, test_ibt)
 
 	expect_fault = true;
 	expect_code = CTRL_PROTECTION_ERRORCODE_ENDBRANCH;
+	expect_reason = IV_CTRL_PROTECTION_EXCEPTION;
 	do_call(should_not_work, 1);
 	zassert_unreachable("should_not_work did not fault");
 }
+#endif /* CONFIG_X86_CET_IBT */
 
 ZTEST_SUITE(cet, NULL, NULL, NULL, NULL, NULL);
diff --git a/tests/arch/x86/cet/testcase.yaml b/tests/arch/x86/cet/testcase.yaml
index ce4d5630b92b9..3cb9bc00043a2 100644
--- a/tests/arch/x86/cet/testcase.yaml
+++ b/tests/arch/x86/cet/testcase.yaml
@@ -17,3 +17,14 @@ tests:
     filter: CONFIG_X86_64
     extra_configs:
       - CONFIG_X86_CET_IBT=y
+  arch.x86.cet.kernel.shstk64:
+    arch_allow: x86
+    filter: CONFIG_X86_64
+    extra_configs:
+      - CONFIG_HW_SHADOW_STACK=y
+  arch.x86.cet.kernel.shstk32:
+    arch_allow: x86
+    filter: not CONFIG_X86_64
+    extra_configs:
+      - CONFIG_DEBUG_COREDUMP=y
+      - CONFIG_HW_SHADOW_STACK=y