i386: Change indirect_return to function type attribute

In

struct ucontext;
typedef struct ucontext ucontext_t;

extern int (*bar) (ucontext_t *__restrict __oucp,
                   const ucontext_t *__restrict __ucp)
  __attribute__((__indirect_return__));

extern int res;

void
foo (ucontext_t *oucp, ucontext_t *ucp)
{
  res = bar (oucp, ucp);
}

bar() may return via indirect branch.  This patch changes indirect_return
to type attribute to allow indirect_return attribute on variable or type
of function pointer so that ENDBR can be inserted after call to bar().

gcc/

	PR target/86560
	* config/i386/i386.c (rest_of_insert_endbranch): Lookup
	indirect_return as function type attribute.
	(ix86_attribute_table): Change indirect_return to function
	type attribute.
	* doc/extend.texi: Update indirect_return attribute.

gcc/testsuite/

	PR target/86560
	* gcc.target/i386/pr86560-1.c: New test.
	* gcc.target/i386/pr86560-2.c: Likewise.
	* gcc.target/i386/pr86560-3.c: Likewise.

From-SVN: r262877

gcc/ChangeLog

@@ -1,3 +1,12 @@
+2018-07-19  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/86560
+	* config/i386/i386.c (rest_of_insert_endbranch): Lookup
+	indirect_return as function type attribute.
+	(ix86_attribute_table): Change indirect_return to function
+	type attribute.
+	* doc/extend.texi: Update indirect_return attribute.
+
 2018-07-19  Aldy Hernandez  <aldyh@redhat.com>
 
 	* wide-int.h (widest2_int): New.

gcc/config/i386/i386.c

@@ -2635,16 +2635,23 @@ rest_of_insert_endbranch (void)
 		{
 		  rtx call = get_call_rtx_from (insn);
 		  rtx fnaddr = XEXP (call, 0);
+		  tree fndecl = NULL_TREE;
 
 		  /* Also generate ENDBRANCH for non-tail call which
 		     may return via indirect branch.  */
-		  if (MEM_P (fnaddr)
-		      && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
-		    {
-		      tree fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
+		  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+		    fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
+		  if (fndecl == NULL_TREE)
+		    fndecl = MEM_EXPR (fnaddr);
 		  if (fndecl
-			  && lookup_attribute ("indirect_return",
-					       DECL_ATTRIBUTES (fndecl)))
+		      && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
+		      && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
+		    fndecl = NULL_TREE;
+		  if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
+		    {
+		      tree fntype = TREE_TYPE (fndecl);
+		      if (lookup_attribute ("indirect_return",
+					    TYPE_ATTRIBUTES (fntype)))
 			need_endbr = true;
 		    }
 		}
@@ -45920,8 +45927,8 @@ static const struct attribute_spec ix86_attribute_table[] =
     ix86_handle_fndecl_attribute, NULL },
   { "function_return", 1, 1, true, false, false, false,
     ix86_handle_fndecl_attribute, NULL },
-  { "indirect_return", 0, 0, true, false, false, false,
-    ix86_handle_fndecl_attribute, NULL },
+  { "indirect_return", 0, 0, false, true, true, false,
+    NULL, NULL },
 
   /* End element.  */
   { NULL, 0, 0, false, false, false, false, NULL, NULL }

gcc/doc/extend.texi

@@ -5889,8 +5889,9 @@ foo (void)
 @item indirect_return
 @cindex @code{indirect_return} function attribute, x86
 
-The @code{indirect_return} attribute on a function is used to inform
-the compiler that the function may return via indirect branch.
+The @code{indirect_return} attribute can be applied to a function,
+as well as variable or type of function pointer to inform the
+compiler that the function may return via indirect branch.
 
 @end table
 

gcc/testsuite/ChangeLog

@@ -1,3 +1,10 @@
+2018-07-19  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR target/86560
+	* gcc.target/i386/pr86560-1.c: New test.
+	* gcc.target/i386/pr86560-2.c: Likewise.
+	* gcc.target/i386/pr86560-3.c: Likewise.
+
 2018-07-19  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
 
 	* gfortran.dg/max_fmax_aarch64.f90: New test.

gcc/testsuite/gcc.target/i386/pr86560-1.c

@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+extern int (*bar) (struct ucontext *)
+  __attribute__((__indirect_return__));
+
+extern int res;
+
+void
+foo (struct ucontext *oucp)
+{
+  res = bar (oucp);
+}

gcc/testsuite/gcc.target/i386/pr86560-2.c

@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+typedef int (*bar_p) (struct ucontext *)
+  __attribute__((__indirect_return__));
+
+extern int res;
+
+void
+foo (bar_p bar, struct ucontext *oucp)
+{
+  res = bar (oucp);
+}

gcc/testsuite/gcc.target/i386/pr86560-3.c

@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fcf-protection" } */
+/* { dg-final { scan-assembler-times {\mendbr} 2 } } */
+
+struct ucontext;
+
+extern int (*bar) (struct ucontext *);
+
+extern int res;
+
+void
+foo (struct ucontext *oucp)
+{
+  int (*f) (struct ucontext *) __attribute__((__indirect_return__))
+    = bar;
+  res = f (oucp);
+}