kernel: Automatically set up HW shadow stack for thread stacks

This patch modifies thread stack macros (such as K_KERNEL_STACK_DECLARE
or K_KERNEL_STACK_ARRAY_DECLARE) to also create a HW shadow stack (when
CONFIG_HW_SHADOW_STACK=y), as well as define a pairing between the
thread stack (or thread stack array) and the shadow stack (or shadow
stack array).

This pairing, which currently is simply an array of pairs (stack,
shadow_stack) is searched during thread setup to find the corresponding
shadow stack and attach it to the thread. If linear search on this array
proves to be a performance issue, the actual structure can be revisited.

To define the size of the shadow stack for a given stack, the stack size
is used. A new Kconfig, CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE is used
to define how big the shadow stack is compared to the stack. Note that
this size is in *addition* to the stack size. To avoid some shadow
stacks becoming too small, CONFIG_HW_SHADOW_STACK_MIN_SIZE is used to
define a minimum size. Note that after this size is defined, platform
restrictions on the size of the shadow stack are still applied.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>

Author: edersondisouza

include/zephyr/kernel/thread_stack.h

@@ -112,6 +112,79 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @{
  */
 
+#ifdef CONFIG_HW_SHADOW_STACK
+#define k_thread_hw_shadow_stack_t arch_thread_hw_shadow_stack_t
+
+#define K_THREAD_HW_SHADOW_STACK_SIZE(size_) \
+	MAX(ROUND_UP((CONFIG_HW_SHADOW_STACK_PERCENTAGE_SIZE * (size_) / 100), \
+		      CONFIG_X86_CET_SHADOW_STACK_ALIGNMENT), \
+	    CONFIG_HW_SHADOW_STACK_MIN_SIZE)
+
+#define K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, size) \
+	ARCH_THREAD_HW_SHADOW_STACK_DECLARE(__ ## sym ## _shstk, size)
+
+#define K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DECLARE(__ ## sym ## _shstk_arr, \
+						  nmemb, size)
+
+struct _stack_to_hw_shadow_stack {
+	k_thread_stack_t *stack;
+	k_thread_hw_shadow_stack_t *shstk_addr;
+	size_t size;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_DEFINE(sym, size_) \
+	ARCH_THREAD_HW_SHADOW_STACK_DEFINE(__ ## sym ## _shstk, size_); \
+	static const STRUCT_SECTION_ITERABLE(_stack_to_hw_shadow_stack, \
+		sym ## _stack_to_shstk_attach) = { \
+		.stack = sym, \
+		.shstk_addr = __ ## sym ## _shstk, \
+		.size = size_, \
+	}
+
+struct _stack_to_hw_shadow_stack_arr {
+	uintptr_t stack_addr;
+	uintptr_t shstk_addr;
+	size_t stack_size;
+	size_t shstk_size;
+	size_t nmemb;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb_, size_) \
+	ARCH_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(__ ## sym ## _shstk_arr, nmemb_, \
+			K_THREAD_HW_SHADOW_STACK_SIZE(size_)); \
+	static const STRUCT_SECTION_ITERABLE(_stack_to_hw_shadow_stack_arr, \
+		sym ## _stack_to_shstk_attach) = { \
+		.stack_addr = (uintptr_t)sym, \
+		.stack_size = K_KERNEL_STACK_LEN(size_), \
+		.nmemb = nmemb_, \
+		.shstk_addr = (uintptr_t)__ ## sym ## _shstk_arr, \
+		.shstk_size = K_THREAD_HW_SHADOW_STACK_SIZE(size_), \
+	}
+
+#define k_thread_hw_shadow_stack_attach arch_thread_hw_shadow_stack_attach
+
+struct _thread_hw_shadow_stack_static {
+	struct k_thread *thread;
+	k_thread_hw_shadow_stack_t *shstk_addr;
+	size_t size;
+};
+
+#define K_THREAD_HW_SHADOW_STACK_ATTACH(thread_, shstk_addr_, size_) \
+	static const STRUCT_SECTION_ITERABLE(_thread_hw_shadow_stack_static, \
+		thread ## _shstk_attach_static) = { \
+		.thread = thread_, \
+		.shstk_addr = shstk_addr_, \
+		.size = size_, \
+	}
+
+#else
+#define K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, size)
+#define K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, size)
+#define K_THREAD_HW_SHADOW_STACK_DEFINE(sym, size)
+#define K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size_)
+#endif
+
 /**
  * @brief Declare a reference to a thread stack
  *
@@ -122,6 +195,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_DECLARE(sym, size) \
+	K_KERNEL_HW_SHADOW_STACK_DECLARE(sym, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[K_KERNEL_STACK_LEN(size)]
 
@@ -136,6 +210,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[nmemb][K_KERNEL_STACK_LEN(size)]
 
@@ -150,6 +225,7 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_PINNED_STACK_ARRAY_DECLARE(sym, nmemb, size) \
+	K_KERNEL_HW_SHADOW_STACK_ARRAY_DECLARE(sym, nmemb, K_THREAD_HW_SHADOW_STACK_SIZE(size)); \
 	extern struct z_thread_stack_element \
 		sym[nmemb][K_KERNEL_STACK_LEN(size)]
 
@@ -212,7 +288,9 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 
 /**
  * @brief Define a toplevel kernel stack memory region in pinned section
@@ -228,10 +306,14 @@ static inline char *z_stack_ptr_align(char *ptr)
  */
 #if defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 #else
 #define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
-	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_DEFINE(sym, \
+					K_THREAD_HW_SHADOW_STACK_SIZE(size))
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
 /**
@@ -244,7 +326,9 @@ static inline char *z_stack_ptr_align(char *ptr)
  * @param size Size of the stack memory region
  */
 #define K_KERNEL_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
+
 
 /**
  * @brief Define a toplevel array of kernel stack memory regions in pinned section
@@ -261,10 +345,12 @@ static inline char *z_stack_ptr_align(char *ptr)
  */
 #if defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
 #else
 #define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
-	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem); \
+	K_THREAD_HW_SHADOW_STACK_ARRAY_DEFINE(sym, nmemb, size)
 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
 /**
@@ -283,14 +369,6 @@ static inline char *z_stack_ptr_align(char *ptr)
 
 /** @} */
 
-#ifdef CONFIG_HW_SHADOW_STACK
-#define K_THREAD_HW_SHADOW_STACK_DEFINE ARCH_THREAD_HW_SHADOW_STACK_DEFINE
-
-#define k_thread_hw_shadow_stack_t arch_thread_hw_shadow_stack_t
-
-#define k_thread_hw_shadow_stack_attach arch_thread_hw_shadow_stack_attach
-#endif
-
 static inline char *K_KERNEL_STACK_BUFFER(k_thread_stack_t *sym)
 {
 	return (char *)sym + K_KERNEL_STACK_RESERVED;

include/zephyr/linker/common-rom/common-rom-kernel-devices.ld

@@ -102,4 +102,9 @@
 	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
 #endif /* !CONFIG_DEVICE_DEPS_DYNAMIC */
 
+#ifdef CONFIG_HW_SHADOW_STACK
+	ITERABLE_SECTION_ROM(_stack_to_hw_shadow_stack, Z_LINK_ITERABLE_SUBALIGN)
+	ITERABLE_SECTION_ROM(_stack_to_hw_shadow_stack_arr, Z_LINK_ITERABLE_SUBALIGN)
+#endif
+
 #include <device-api-sections.ld>

kernel/Kconfig

@@ -1045,6 +1045,31 @@ config HW_SHADOW_STACK
 	  stack. Indeed, threads need to manually attach a shadow stack to
 	  use this feature. See the documentation for more information.
 
+config HW_SHADOW_STACK_PERCENTAGE_SIZE
+	int "Percentage of the stack size used to define shadow stack size"
+	depends on HW_SHADOW_STACK
+	default 30
+	range 0 100
+	help
+	  This option specifies the percentage of the stack to be used for
+	  shadow stack. The value is a percentage of the total stack size.
+	  The default value is 30%, which means that the shadow stack size
+	  will be 30% of the stack size, in *addition* to the stack size.
+	  used for shadow stack. Note that this size is constrained by the
+	  HW_SHADOW_STACK_MIN_SIZE config option.
+
+config HW_SHADOW_STACK_MIN_SIZE
+	int "Minimum size of the hardware shadow stack"
+	depends on HW_SHADOW_STACK
+	default 256
+	help
+	  This option specifies the minimum size of the hardware shadow stack,
+	  in bytes, so that threads that use minimal stack size can still
+	  have a sensible shadow stack size. The default value is 256 bytes,
+	  which means that the shadow stack size will be at least 256 bytes,
+	  even if the percentage defined via HW_SHADOW_STACK_PERCENTAGE_SIZE
+	  would define a smaller size.
+
 config HW_SHADOW_STACK_ALLOW_REUSE
 	bool "Allow reuse of the shadow stack"
 	depends on HW_SHADOW_STACK

kernel/thread.c

@@ -501,6 +501,70 @@ static char *setup_thread_stack(struct k_thread *new_thread,
 	return stack_ptr;
 }
 
+#ifdef CONFIG_HW_SHADOW_STACK
+static void setup_shadow_stack(struct k_thread *new_thread,
+			 k_thread_stack_t *stack)
+{
+	int ret = -ENOENT;
+
+	STRUCT_SECTION_FOREACH(_stack_to_hw_shadow_stack, stk_to_hw_shstk) {
+		if (stk_to_hw_shstk->stack == stack) {
+			ret = k_thread_hw_shadow_stack_attach(new_thread,
+							      stk_to_hw_shstk->shstk_addr,
+							      stk_to_hw_shstk->size);
+			if (ret != 0) {
+				LOG_ERR("Could not set thread %p shadow stack %p, got error %d",
+					new_thread, stk_to_hw_shstk->shstk_addr, ret);
+				k_panic();
+			}
+			break;
+		}
+	}
+
+	/* Check if the stack isn't in a stack array, and use the corresponding
+	 * shadow stack.
+	 */
+	if (ret != -ENOENT) {
+		return;
+	}
+
+	STRUCT_SECTION_FOREACH(_stack_to_hw_shadow_stack_arr, stk_to_hw_shstk) {
+		if ((uintptr_t)stack >= stk_to_hw_shstk->stack_addr &&
+		    (uintptr_t)stack < stk_to_hw_shstk->stack_addr +
+		    stk_to_hw_shstk->stack_size * stk_to_hw_shstk->nmemb) {
+			/* Now we have to guess which index of the stack array is being used */
+			uintptr_t stack_offset = (uintptr_t)stack - stk_to_hw_shstk->stack_addr;
+			uintptr_t stack_index = stack_offset / stk_to_hw_shstk->stack_size;
+			uintptr_t addr;
+
+			if (stack_index >= stk_to_hw_shstk->nmemb) {
+				LOG_ERR("Could not find shadow stack for thread %p, stack %p",
+					new_thread, stack);
+				k_panic();
+			}
+
+			addr = stk_to_hw_shstk->shstk_addr +
+				stk_to_hw_shstk->shstk_size * stack_index;
+			ret = k_thread_hw_shadow_stack_attach(new_thread,
+							      (arch_thread_hw_shadow_stack_t *)addr,
+							      stk_to_hw_shstk->shstk_size);
+			if (ret != 0) {
+				LOG_ERR("Could not set thread %p shadow stack 0x%lx, got error %d",
+					new_thread, stk_to_hw_shstk->shstk_addr, ret);
+				k_panic();
+			}
+			break;
+		}
+	}
+
+	if (ret == -ENOENT) {
+		LOG_ERR("Could not find shadow stack for thread %p, stack %p",
+			new_thread, stack);
+		k_panic();
+	}
+}
+#endif
+
 /*
  * The provided stack_size value is presumed to be either the result of
  * K_THREAD_STACK_SIZEOF(stack), or the size value passed to the instance
@@ -547,6 +611,10 @@ char *z_setup_new_thread(struct k_thread *new_thread,
 	z_init_thread_base(&new_thread->base, prio, _THREAD_SLEEPING, options);
 	stack_ptr = setup_thread_stack(new_thread, stack, stack_size);
 
+#ifdef CONFIG_HW_SHADOW_STACK
+	setup_shadow_stack(new_thread, stack);
+#endif
+
 #ifdef CONFIG_KERNEL_COHERENCE
 	/* Check that the thread object is safe, but that the stack is
 	 * still cached!