x86: Disallow -mindirect-branch=/-mfunction-return= with -mshstkhjl/indirect/master

Since call and return thunk changes call stack without updating shadow
stack, -mindirect-branch= and -mfunction-return= can't be used with
-mshstk.  Issue an error when they are used with -mshstk.

<immintrin.h> can't include <cetintrin.h> with -mindirect-branch= nor
-mfunction-return=.  The __indirect_branch__ macro is predefined with
-mindirect-branch= and the __function_return__ is is predefined with
-mfunction-return= for <immintrin.h> to check if -mindirect-branch= or
-mfunction-return= are used.

gcc/

	* config/i386/i386-c.c (ix86_target_macros_internal): Define
	__indirect_branch__ for -mindirect-branch=.  Define
	__function_return__ for -mfunction-return=.
	* config/i386/i386.c (ix86_set_indirect_branch_type): Disallow
	-mindirect-branch=/-mfunction-return= with -mshstk.
	* config/i386/immintrin.h: Don't include <cetintrin.h> if
	-mindirect-branch= or -mfunction-return= are used.
	* doc/invoke.texi: Document -mshstk are incompatible with
	-mindirect-branch= and -mfunction-return=.

gcc/testsuite/

	* gcc.target/i386/indirect-thunk-8.c: New test.
	* gcc.target/i386/indirect-thunk-9.c: Likewise.
	* gcc.target/i386/indirect-thunk-attr-9.c: Likewise.
	* gcc.target/i386/ret-thunk-17.c: Likewise.
	* gcc.target/i386/ret-thunk-18.c: Likewise.
	* gcc.target/i386/ret-thunk-19.c: Likewise.

10 files changed, 76 insertions, 0 deletions

diff --git a/gcc/config/i386/i386-c.c b/gcc/config/i386/i386-c.c
index 78dd65785f6..c2795cd1341 100644
--- a/gcc/config/i386/i386-c.c
+++ b/gcc/config/i386/i386-c.c
@@ -495,6 +495,10 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
       def_or_undef (parse_in, "__iamcu");
       def_or_undef (parse_in, "__iamcu__");
     }
+  if (ix86_indirect_branch != indirect_branch_keep)
+    def_or_undef (parse_in, "__indirect_branch__");
+  if (ix86_function_return != indirect_branch_keep)
+    def_or_undef (parse_in, "__function_return__");
 }
 
 
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index e69135d7191..d35d5ec991d 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -5832,6 +5832,12 @@ ix86_set_indirect_branch_type (tree fndecl)
 	}
       else
 	cfun->machine->indirect_branch_type = ix86_indirect_branch;
+
+      /* -mindirect-branch= and -mshstk are not compatible.  */
+      if (TARGET_SHSTK
+	  && cfun->machine->indirect_branch_type != indirect_branch_keep)
+	error ("%<-mindirect-branch=%> and %<-mshstk%> are "
+	       "not compatible");
     }
 
   if (cfun->machine->function_return_type == indirect_branch_unset)
@@ -5857,6 +5863,12 @@ ix86_set_indirect_branch_type (tree fndecl)
 	}
       else
 	cfun->machine->function_return_type = ix86_function_return;
+
+      /* -mfunction-return= and -mshstk are not compatible.  */
+      if (TARGET_SHSTK
+	  && cfun->machine->function_return_type != indirect_branch_keep)
+	    error ("%<-mfunction-return=%> and %<-mshstk%> are "
+		   "not compatible");
     }
 }
 
diff --git a/gcc/config/i386/immintrin.h b/gcc/config/i386/immintrin.h
index a5ad8af32b0..c64e25a981f 100644
--- a/gcc/config/i386/immintrin.h
+++ b/gcc/config/i386/immintrin.h
@@ -102,7 +102,11 @@
 
 #include <xtestintrin.h>
 
+/* -mindirect-branch= and -mfunction-return= are not compatible with
+   -mshstk.  */
+#if !defined __indirect_branch__ && !defined __function_return__
 #include <cetintrin.h>
+#endif
 
 #include <gfniintrin.h>
 
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index d16006e653a..91e2f51e85a 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -26851,6 +26851,10 @@ to external call and return thunk provided in a separate object file.
 You can control this behavior for a specific function by using the
 function attribute @code{indirect_branch}.  @xref{Function Attributes}.
 
+Note that @option{-mindirect-branch=} is incompatible with
+@option{-mshstk} since call and return thunk changes call stack
+without updating shadow stack.
+
 @item -mfunction-return=@var{choice}
 @opindex -mfunction-return
 Convert function return with @var{choice}.  The default is @samp{keep},
@@ -26862,6 +26866,10 @@ object file.  You can control this behavior for a specific function by
 using the function attribute @code{function_return}.
 @xref{Function Attributes}.
 
+Note that @option{-mfunction-return=} is incompatible with
+@option{-mshstk} since call and return thunk changes call stack
+without updating shadow stack.
+
 @item -mindirect-branch-register
 @opindex -mindirect-branch-register
 Force indirect call and jump via register.
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c
new file mode 100644
index 00000000000..e588bb3843a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c
@@ -0,0 +1,7 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mindirect-branch=thunk -mfunction-return=keep -mshstk" } */
+
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=' and '-mshstk' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c
new file mode 100644
index 00000000000..1ecc07a530d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mindirect-branch=thunk" } */
+
+#include <x86intrin.h>
+
+void
+bar (void)
+{
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c
new file mode 100644
index 00000000000..d1fc429f485
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mindirect-branch=keep -mfunction-return=keep -mshstk" } */
+
+__attribute__ ((indirect_branch("thunk")))
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=' and '-mshstk' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-17.c b/gcc/testsuite/gcc.target/i386/ret-thunk-17.c
new file mode 100644
index 00000000000..b1a45c1d94f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-17.c
@@ -0,0 +1,7 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mfunction-return=thunk -mindirect-branch=keep -mshstk" } */
+
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=' and '-mshstk' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-18.c b/gcc/testsuite/gcc.target/i386/ret-thunk-18.c
new file mode 100644
index 00000000000..09344299246
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-18.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mfunction-return=keep -mindirect-branch=keep -mshstk" } */
+
+__attribute__ ((function_return("thunk")))
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=' and '-mshstk' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-19.c b/gcc/testsuite/gcc.target/i386/ret-thunk-19.c
new file mode 100644
index 00000000000..e9028798903
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-19.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mfunction-return=thunk" } */
+
+#include <x86intrin.h>
+
+void
+bar (void)
+{
+}