[pull] master from ruby:master

configure.ac

@@ -2822,6 +2822,12 @@ AS_CASE([$coroutine_type], [yes|''], [
         [powerpc64le-freebsd*], [
             coroutine_type=ppc64le
         ],
+        [riscv64-freebsd*], [
+            coroutine_type=riscv64
+        ],
+        [arm*-freebsd*], [
+            coroutine_type=arm32
+        ],
         [x86_64-netbsd*], [
             coroutine_type=amd64
         ],

cont.c

@@ -3738,6 +3738,13 @@ ruby_Init_Continuation_body(void)
     rb_undef_method(CLASS_OF(rb_cContinuation), "new");
     rb_define_method(rb_cContinuation, "call", rb_cont_call, -1);
     rb_define_method(rb_cContinuation, "[]", rb_cont_call, -1);
+#ifdef COROUTINE_SHADOW_STACK
+    if (coroutine_shadow_stack_enabled()) {
+        /* Continuations cannot restore previously unwound shadow stack frames. */
+        rb_define_global_function("callcc", rb_f_notimplement, 0);
+        return;
+    }
+#endif
     rb_define_global_function("callcc", rb_callcc, 0);
 }
 

coroutine/amd64/Context.S

@@ -11,14 +11,18 @@
 
 #define TOKEN_PASTE(x,y) x##y
 
+#if defined(__linux__) && defined(__CET__) && (__CET__ & 0x02) != 0
+#define COROUTINE_SHADOW_STACK 1
+#endif
+
 .text
 
 .globl PREFIXED_SYMBOL(coroutine_transfer)
 PREFIXED_SYMBOL(coroutine_transfer):
 
 #if defined(__CET__) && (__CET__ & 0x01) != 0
-        /* IBT landing pad */
-        endbr64
+	/* IBT landing pad */
+	endbr64
 #endif
 
 	# Make space on the stack for 6 registers:
@@ -31,13 +35,39 @@ PREFIXED_SYMBOL(coroutine_transfer):
 	movq %r13, 16(%rsp)
 	movq %r14, 8(%rsp)
 	movq %r15, (%rsp)
+#if defined(COROUTINE_SHADOW_STACK)
+	# A zero target SSP means shadow stacks are not enabled at runtime.
+	movq (%rsi), %rax
+	cmpq $0, (%rax)
+	je 1f
+
+	# Save the source shadow stack pointer:
+	rdsspq %rbp
+	pushq %rbp
+	jmp 2f
+1:
+	pushq $0
+2:
+#endif
 
 	# Save caller stack pointer:
 	movq %rsp, (%rdi)
 
 	# Restore callee stack pointer:
 	movq (%rsi), %rsp
 
+#if defined(COROUTINE_SHADOW_STACK)
+	# Restore the destination shadow stack pointer:
+	popq %rbp
+	testq %rbp, %rbp
+	jz 3f
+	rstorssp -8(%rbp)
+
+	# Save the source shadow stack pointer token:
+	saveprevssp
+3:
+#endif
+
 	# Restore callee state
 	movq 40(%rsp), %rbp
 	movq 32(%rsp), %rbx
@@ -55,6 +85,44 @@ PREFIXED_SYMBOL(coroutine_transfer):
 	# We pop the return address and jump to it
 	ret
 
+#if defined(COROUTINE_SHADOW_STACK)
+.globl PREFIXED_SYMBOL(coroutine_initialize_shadow_stack)
+PREFIXED_SYMBOL(coroutine_initialize_shadow_stack):
+#if defined(__CET__) && (__CET__ & 0x01) != 0
+	/* IBT landing pad */
+	endbr64
+#endif
+	# Save the current SSP and switch to the new shadow stack. The restore
+	# token created by map_shadow_stack is immediately below the supplied SSP.
+	rdsspq %r8
+	rstorssp -8(%rdi)
+	saveprevssp
+
+	# CALL is the only way to place an arbitrary return address on a shadow
+	# stack without enabling WRSS. Its return address is the trampoline below.
+	call 4f
+
+.globl PREFIXED_SYMBOL(coroutine_start_trampoline)
+PREFIXED_SYMBOL(coroutine_start_trampoline):
+#if defined(__CET__) && (__CET__ & 0x01) != 0
+	/* IBT landing pad */
+	endbr64
+#endif
+	jmp *%r12
+
+4:
+	# Remove CALL's entry from the normal stack while retaining its shadow
+	# stack entry, then remember the resulting SSP for the new coroutine.
+	popq %rax
+	rdsspq %rax
+
+	# Restore the original shadow stack. This leaves a restore token for the
+	# newly initialized stack immediately below the SSP returned in %rax.
+	rstorssp -8(%r8)
+	saveprevssp
+	ret
+#endif
+
 #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__ELF__)
 .section .note.GNU-stack,"",%progbits
 #endif
@@ -67,10 +135,11 @@ PREFIXED_SYMBOL(coroutine_transfer):
 # define IBT_FLAG 0x00
 #endif
 
-/* We do _NOT_ support CET shadow-stack. Do _not_ add the property for
- * this to the Context.o object. If you require CET shadow-stack support,
- * for now, consider building with --with-coroutine=ucontext */
-#define SHSTK_FLAG 0x00
+#if defined(COROUTINE_SHADOW_STACK)
+# define SHSTK_FLAG 0x02
+#else
+# define SHSTK_FLAG 0x00
+#endif
 
 .pushsection .note.gnu.property, "a"
 .p2align 3

coroutine/amd64/Context.h

@@ -48,6 +48,65 @@ enum {COROUTINE_REGISTERS = 6};
 #include <sanitizer/tsan_interface.h>
 #endif
 
+#if defined(__linux__) && defined(__CET__) && (__CET__ & 0x02) != 0
+#define COROUTINE_SHADOW_STACK
+
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#ifndef ARCH_SHSTK_STATUS
+#define ARCH_SHSTK_STATUS 0x5005
+#endif
+
+#ifndef ARCH_SHSTK_SHSTK
+#define ARCH_SHSTK_SHSTK (1UL << 0)
+#endif
+
+#ifndef SYS_map_shadow_stack
+#ifdef __NR_map_shadow_stack
+#define SYS_map_shadow_stack __NR_map_shadow_stack
+#else
+#define SYS_map_shadow_stack 453
+#endif
+#endif
+
+#ifndef SHADOW_STACK_SET_TOKEN
+#define SHADOW_STACK_SET_TOKEN (1UL << 0)
+#endif
+
+void *coroutine_initialize_shadow_stack(void *shadow_stack_pointer);
+COROUTINE coroutine_start_trampoline(void);
+
+static inline int coroutine_shadow_stack_enabled(void)
+{
+    unsigned long features = 0;
+
+    if (syscall(SYS_arch_prctl, ARCH_SHSTK_STATUS, &features) != 0) {
+        return 0;
+    }
+
+    return (features & ARCH_SHSTK_SHSTK) != 0;
+}
+
+static inline void *coroutine_allocate_shadow_stack(size_t size)
+{
+    void *base = (void *)syscall(
+        SYS_map_shadow_stack,
+        0,
+        size,
+        SHADOW_STACK_SET_TOKEN
+    );
+
+    if (base == MAP_FAILED) {
+        abort();
+    }
+
+    return base;
+}
+#endif
+
 struct coroutine_context
 {
     void **stack_pointer;
@@ -66,6 +125,11 @@ struct coroutine_context
      * implicit fiber, owned by TSan; must not be destroyed). */
     int tsan_fiber_owned;
 #endif
+
+#if defined(COROUTINE_SHADOW_STACK)
+    void *shadow_stack;
+    size_t shadow_stack_size;
+#endif
 };
 
 typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self);
@@ -78,6 +142,11 @@ static inline void coroutine_initialize_main(struct coroutine_context * context)
     context->tsan_fiber = __tsan_get_current_fiber();
     context->tsan_fiber_owned = 0;
 #endif
+
+#if defined(COROUTINE_SHADOW_STACK)
+    context->shadow_stack = NULL;
+    context->shadow_stack_size = 0;
+#endif
 }
 
 static inline void coroutine_initialize(
@@ -88,6 +157,11 @@ static inline void coroutine_initialize(
 ) {
     assert(start && stack && size >= 1024);
 
+#if defined(COROUTINE_SHADOW_STACK)
+    void *shadow_stack_pointer = NULL;
+    void *entry = (void *)(uintptr_t)start;
+#endif
+
 #if defined(COROUTINE_SANITIZE_ADDRESS)
     context->fake_stack = NULL;
     context->stack_base = stack;
@@ -99,15 +173,44 @@ static inline void coroutine_initialize(
     context->tsan_fiber_owned = 1;
 #endif
 
+#if defined(COROUTINE_SHADOW_STACK)
+    if (coroutine_shadow_stack_enabled()) {
+        size_t shadow_stack_size = (size + 7) & ~(size_t)7;
+
+        context->shadow_stack = coroutine_allocate_shadow_stack(shadow_stack_size);
+        context->shadow_stack_size = shadow_stack_size;
+        shadow_stack_pointer = coroutine_initialize_shadow_stack(
+            (char *)context->shadow_stack + shadow_stack_size
+        );
+        entry = (void *)(uintptr_t)coroutine_start_trampoline;
+    } else {
+        context->shadow_stack = NULL;
+        context->shadow_stack_size = 0;
+    }
+#endif
+
     // Stack grows down. Force 16-byte alignment.
     char * top = (char*)stack + size;
     context->stack_pointer = (void**)((uintptr_t)top & ~0xF);
 
     *--context->stack_pointer = NULL;
+#if defined(COROUTINE_SHADOW_STACK)
+    *--context->stack_pointer = entry;
+#else
     *--context->stack_pointer = (void*)(uintptr_t)start;
+#endif
 
     context->stack_pointer -= COROUTINE_REGISTERS;
     memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+
+#if defined(COROUTINE_SHADOW_STACK)
+    if (shadow_stack_pointer) {
+        /* coroutine_start_trampoline jumps to the start function in r12. */
+        context->stack_pointer[3] = (void *)(uintptr_t)start;
+    }
+
+    *--context->stack_pointer = shadow_stack_pointer;
+#endif
 }
 
 struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target);
@@ -126,6 +229,13 @@ static inline void coroutine_destroy(struct coroutine_context * context)
         context->tsan_fiber_owned = 0;
     }
 #endif
+
+#if defined(COROUTINE_SHADOW_STACK)
+    if (context->shadow_stack) {
+        munmap(context->shadow_stack, context->shadow_stack_size);
+        context->shadow_stack = NULL;
+    }
+#endif
 }
 
 #endif /* COROUTINE_AMD64_CONTEXT_H */

coroutine/arm64/Context.S

@@ -141,20 +141,20 @@ PREFIXED_SYMBOL(coroutine_transfer):
 #    define PAC_FLAG 0
 #  endif
 
-  # The note section format is described by Note Section in Chapter 5
-  # of "System V Application Binary Interface, Edition 4.1".
-  .pushsection .note.gnu.property, "a"
-  .p2align 3
-  .long 0x4        /* Name size ("GNU\0") */
-  .long 0x10       /* Descriptor size */
-  .long 0x5        /* Type: NT_GNU_PROPERTY_TYPE_0 */
-  .asciz "GNU"     /* Name */
-  # Begin descriptor
-  .long 0xc0000000 /* Property type: GNU_PROPERTY_AARCH64_FEATURE_1_AND */
-  .long 0x4        /* Property size */
-  .long (BTI_FLAG|PAC_FLAG)
-  .long 0x0        /* 8-byte alignment padding */
-  # End descriptor
-  .popsection
+	# The note section format is described by Note Section in Chapter 5
+	# of "System V Application Binary Interface, Edition 4.1".
+	.pushsection .note.gnu.property, "a"
+	.p2align 3
+	.long 0x4        /* Name size ("GNU\0") */
+	.long 0x10       /* Descriptor size */
+	.long 0x5        /* Type: NT_GNU_PROPERTY_TYPE_0 */
+	.asciz "GNU"     /* Name */
+	# Begin descriptor
+	.long 0xc0000000 /* Property type: GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+	.long 0x4        /* Property size */
+	.long (BTI_FLAG|PAC_FLAG)
+	.long 0x0        /* 8-byte alignment padding */
+	# End descriptor
+	.popsection
 #endif
 #endif

coroutine/ppc/Context.S

@@ -46,9 +46,12 @@ PREFIXED_SYMBOL(coroutine_transfer):
 	stw r13,72(r1)
 
 	; Save return address
-	; Possibly should rather be saved into linkage area, see libphobos and IBM docs
 	stw r0,76(r1)
 
+	; Save nonvolatile CR fields in the caller linkage area
+	mfcr r0
+	stw r0,84(r1)
+
 	; Save stack pointer to first argument
 	stw r1,0(r3)
 
@@ -82,6 +85,10 @@ PREFIXED_SYMBOL(coroutine_transfer):
 	; Set LR
 	mtlr r0
 
+	; Restore nonvolatile CR2-CR4
+	lwz r0,84(r1)
+	mtcrf 56,r0
+
 	; Pop stack frame
 	addi r1,r1,80
 

coroutine/ppc/Context.h

@@ -14,7 +14,7 @@
 enum {
   COROUTINE_REGISTERS =
   20  /* 19 general purpose registers (r13-r31) and 1 return address */
-  + 4  /* space for fiber_entry() to store the link register */
+  + 4  /* caller linkage area, including the condition register save slot */
 };
 
 struct coroutine_context