Fix PR gdb/19250: ptrace prototype is not detected properly in C++ modeusers/palves/ptrace-detection

The ptrace args/return types detection doesn't work properly in C++
mode, on non-GNU/Linux hosts.

For example, on gcc70 (NetBSD 5.1), where the prototype is:

 int ptrace(int, __pid_t, void*, int);

configure misdetects it as:

 $ grep PTRACE_TYPE config.h
 #define PTRACE_TYPE_ARG1 int
 #define PTRACE_TYPE_ARG3 int *
 #define PTRACE_TYPE_ARG4 int
 /* #undef PTRACE_TYPE_ARG5 */
 #define PTRACE_TYPE_RET int

resulting in:

 ../../src/gdb/amd64bsd-nat.c: In function 'void amd64bsd_fetch_inferior_registers(target_ops*, regcache*, int)':
 ../../src/gdb/amd64bsd-nat.c:56: warning: dereferencing type-punned pointer will break strict-aliasing rules
 ../../src/gdb/amd64bsd-nat.c: In function 'void amd64bsd_store_inferior_registers(target_ops*, regcache*, int)':
 ../../src/gdb/amd64bsd-nat.c:104: warning: dereferencing type-punned pointer will break strict-aliasing rules
 ../../src/gdb/amd64bsd-nat.c:110: warning: dereferencing type-punned pointer will break strict-aliasing rules

The strategy used to detect ptrace argument types is to re-declare the
ptrace function with various argument combinations.  If the we get the
prototype right, the test program compiles successfully.  If we get it
wrong, the compiler errors out and we keep trying.  This relies on the
fact that a function can't be re-declared with different arguments in
C.

This is not working in C++ mode, because we miss making the ptrace
declaration extern "C", resulting in simply declaring a ptrace
overload, which always succeeds to compile, and then the first
arguments combination is always considered the right one.

The fix is thus to use extern "C" to re-declare ptrace.  Note this
requires moving the declaration outside of main, to the global scope,
because local extern "C" declarations are not valid (and fail to
compile).

That alone isn't sufficient, however.  The next problem is that the
return type detection fails.  For example, on FreeBSD, ptrace returns
'int', but we misdetect it as 'long'.  The error for the failing test
for the return type is, on FreeBSD:

 configure:12453: /usr/local/bin/g++48 -c -pipe -DRL_NO_COMPAT -Wno-unused-function -Wno-unused-variable  -g -DLIBICONV_PLUG -g -fno-strict-aliasing  -DLIBICONV_PLUG  conftest.cpp >&5
 conftest.cpp:166:22: error: declaration of C function 'int ptrace()' conflicts with
  EXTERN_C int ptrace ();
		       ^
 In file included from conftest.cpp:154:0:
 /usr/include/sys/ptrace.h:185:5: error: previous declaration 'int ptrace(int, pid_t, caddr_t, int)' here
  int ptrace(int _request, pid_t _pid, caddr_t _addr, int _data);
      ^
 configure:12453: $? = 1
 configure: failed program was:
 ....
 | EXTERN_C int ptrace ();
 |
 | int
 | main ()
 | {
 |
 |   ;
 |   return 0;
 | }
 configure:12462: result: long
 configure:12470: checking types of arguments for ptrace

The problem is that while in C "int foo()" means the args to foo are
unspecified, "int foo()" in C++, even with extern "C", is equivalent
to "int foo(void)".

The fix for that is to make the return type detection another testing
axis in the big loop that probes the arguments' types.

Confirmed that this fixes the NetBSD 5.1 build.  Also tested by
hacking F23's (GNU/Linux) sys/ptrace.h to several of the different
ptrace prototypes, including the 5 arguments variants, and confirming
that the expected values end up in config.h.

gdb/ChangeLog:
2016-04-15  Pedro Alves  <palves@redhat.com>

	PR gdb/19250
	* ptrace.m4 (GDB_AC_PTRACE): Use extern "C" in C++ mode.  In
	ptrace tests, declare the ptrace prototype outside main.  Replace
	gdb_cv_func_ptrace_ret and gdb_cv_func_ptrace_proto by a single
	variable holding return and argument types.  Make return type
	detection just another probing axis.
	* configure: Regenerate.

gdb/gdbserver/ChangeLog:
2016-04-15  Pedro Alves  <palves@redhat.com>

	PR gdb/19250
	* configure: Regenerate.

5 files changed, 177 insertions, 194 deletions

diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index c83cd02bcd1..ba220bcea77 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,5 +1,15 @@
 2016-04-15  Pedro Alves  <palves@redhat.com>
 
+	PR gdb/19250
+	* ptrace.m4 (GDB_AC_PTRACE): Use extern "C" in C++ mode.  In
+	ptrace tests, declare the ptrace prototype outside main.  Replace
+	gdb_cv_func_ptrace_ret and gdb_cv_func_ptrace_proto by a single
+	variable holding return and argument types.  Make return type
+	detection just another probing axis.
+	* configure: Regenerate.
+
+2016-04-15  Pedro Alves  <palves@redhat.com>
+
 	* ada-lang.c (ada_lookup_struct_elt_type): Constify 'type_str' and
 	'name_str' locals.
 
diff --git a/gdb/configure b/gdb/configure
index b523debe484..4cad6c9ae6f 100755
--- a/gdb/configure
+++ b/gdb/configure
@@ -12386,6 +12386,12 @@ gdb_ptrace_headers='
 #if HAVE_UNISTD_H
 # include <unistd.h>
 #endif
+
+#ifdef __cplusplus
+#  define EXTERN_C extern "C"
+#else
+#  define EXTERN_C extern
+#endif
 '
 # There is no point in checking if we don't have a prototype.
 ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
@@ -12403,161 +12409,135 @@ if test $ac_have_decl = 1; then :
 
 else
 
-  : ${gdb_cv_func_ptrace_ret='int'}
-  : ${gdb_cv_func_ptrace_args='int,int,long,long'}
+  : ${gdb_cv_func_ptrace_proto='int,int,int,long,long'}
 
 fi
 
-# Check return type.  Varargs (used on GNU/Linux) conflict with the
-# empty argument list, so check for that explicitly.
+
+# GNU/Linux uses a varargs prototype, so check for that explicitly.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking return type of ptrace" >&5
 $as_echo_n "checking return type of ptrace... " >&6; }
-if test "${gdb_cv_func_ptrace_ret+set}" = set; then :
+if test "${gdb_cv_func_ptrace_proto+set}" = set; then :
   $as_echo_n "(cached) " >&6
 else
+
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C long ptrace (enum __ptrace_request, ...);
+
 int
 main ()
 {
-extern long ptrace (enum __ptrace_request, ...);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_ret='long'
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$gdb_ptrace_headers
-int
-main ()
-{
-extern int ptrace ();
+
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_ret='int'
-else
-  gdb_cv_func_ptrace_ret='long'
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+  gdb_cv_func_ptrace_proto='long,enum __ptrace_request,int,long,long'
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_ret" >&5
-$as_echo "$gdb_cv_func_ptrace_ret" >&6; }
 
-cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_RET $gdb_cv_func_ptrace_ret
-_ACEOF
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_proto" >&5
+$as_echo "$gdb_cv_func_ptrace_proto" >&6; }
 
-# Check argument types.
+# Test all possible return and argument types combinations.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking types of arguments for ptrace" >&5
 $as_echo_n "checking types of arguments for ptrace... " >&6; }
-if test "${gdb_cv_func_ptrace_args+set}" = set; then :
+if test "${gdb_cv_func_ptrace_proto+set}" = set; then :
   $as_echo_n "(cached) " >&6
 else
 
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$gdb_ptrace_headers
-int
-main ()
-{
-extern long ptrace (enum __ptrace_request, ...);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
-else
 
-for gdb_arg1 in 'int' 'long'; do
- for gdb_arg2 in 'pid_t' 'int' 'long'; do
-  for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
-   for gdb_arg4 in 'int' 'long' 'void *'; do
+# Provide a safe default value.
+gdb_cv_func_ptrace_proto='int,int,int,long,long'
+
+for gdb_ret in 'int' 'long'; do
+ for gdb_arg1 in 'int' 'long'; do
+  for gdb_arg2 in 'pid_t' 'int' 'long'; do
+   for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
+    for gdb_arg4 in 'int' 'long' 'void *'; do
      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C $gdb_ret
+  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
+
 int
 main ()
 {
 
-extern $gdb_cv_func_ptrace_ret
-  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
-
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
-    break 4;
+  gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
+       break 5;
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-    for gdb_arg5 in 'int *' 'int' 'long'; do
-     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+     for gdb_arg5 in 'int *' 'int' 'long'; do
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C $gdb_ret
+  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
+
 int
 main ()
 {
 
-extern $gdb_cv_func_ptrace_ret
-  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
-
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-
-gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
-    break 5;
+  gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
+        break 6;
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+     done
     done
    done
   done
  done
 done
-# Provide a safe default value.
-: ${gdb_cv_func_ptrace_args='int,int,long,long'}
+
 
 fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_args" >&5
-$as_echo "$gdb_cv_func_ptrace_args" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_proto" >&5
+$as_echo "$gdb_cv_func_ptrace_proto" >&6; }
+
 ac_save_IFS=$IFS; IFS=','
-set dummy `echo "$gdb_cv_func_ptrace_args" | sed 's/\*/\*/g'`
+set dummy `echo "$gdb_cv_func_ptrace_proto" | sed 's/\*/\*/g'`
 IFS=$ac_save_IFS
 shift
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG1 $1
+#define PTRACE_TYPE_RET $1
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PTRACE_TYPE_ARG1 $2
 _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG3 $3
+#define PTRACE_TYPE_ARG3 $4
 _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG4 $4
+#define PTRACE_TYPE_ARG4 $5
 _ACEOF
 
-if test -n "$5"; then
+if test -n "$6"; then
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG5 $5
+#define PTRACE_TYPE_ARG5 $6
 _ACEOF
 
 fi
diff --git a/gdb/gdbserver/ChangeLog b/gdb/gdbserver/ChangeLog
index 44c4f129326..5f27aac0d75 100644
--- a/gdb/gdbserver/ChangeLog
+++ b/gdb/gdbserver/ChangeLog
@@ -1,3 +1,8 @@
+2016-04-15  Pedro Alves  <palves@redhat.com>
+
+	PR gdb/19250
+	* configure: Regenerate.
+
 2016-04-13  Antoine Tremblay  <antoine.tremblay@ericsson.com>
 
 	* linux-aarch64-low.c (aarch64_emit_add): Switch x1 and x0.
diff --git a/gdb/gdbserver/configure b/gdb/gdbserver/configure
index bb01922c2c1..5ceec386f8b 100755
--- a/gdb/gdbserver/configure
+++ b/gdb/gdbserver/configure
@@ -6065,6 +6065,12 @@ gdb_ptrace_headers='
 #if HAVE_UNISTD_H
 # include <unistd.h>
 #endif
+
+#ifdef __cplusplus
+#  define EXTERN_C extern "C"
+#else
+#  define EXTERN_C extern
+#endif
 '
 # There is no point in checking if we don't have a prototype.
 ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
@@ -6082,161 +6088,135 @@ if test $ac_have_decl = 1; then :
 
 else
 
-  : ${gdb_cv_func_ptrace_ret='int'}
-  : ${gdb_cv_func_ptrace_args='int,int,long,long'}
+  : ${gdb_cv_func_ptrace_proto='int,int,int,long,long'}
 
 fi
 
-# Check return type.  Varargs (used on GNU/Linux) conflict with the
-# empty argument list, so check for that explicitly.
+
+# GNU/Linux uses a varargs prototype, so check for that explicitly.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking return type of ptrace" >&5
 $as_echo_n "checking return type of ptrace... " >&6; }
-if test "${gdb_cv_func_ptrace_ret+set}" = set; then :
+if test "${gdb_cv_func_ptrace_proto+set}" = set; then :
   $as_echo_n "(cached) " >&6
 else
+
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C long ptrace (enum __ptrace_request, ...);
+
 int
 main ()
 {
-extern long ptrace (enum __ptrace_request, ...);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_ret='long'
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$gdb_ptrace_headers
-int
-main ()
-{
-extern int ptrace ();
+
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_ret='int'
-else
-  gdb_cv_func_ptrace_ret='long'
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+  gdb_cv_func_ptrace_proto='long,enum __ptrace_request,int,long,long'
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_ret" >&5
-$as_echo "$gdb_cv_func_ptrace_ret" >&6; }
 
-cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_RET $gdb_cv_func_ptrace_ret
-_ACEOF
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_proto" >&5
+$as_echo "$gdb_cv_func_ptrace_proto" >&6; }
 
-# Check argument types.
+# Test all possible return and argument types combinations.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking types of arguments for ptrace" >&5
 $as_echo_n "checking types of arguments for ptrace... " >&6; }
-if test "${gdb_cv_func_ptrace_args+set}" = set; then :
+if test "${gdb_cv_func_ptrace_proto+set}" = set; then :
   $as_echo_n "(cached) " >&6
 else
 
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$gdb_ptrace_headers
-int
-main ()
-{
-extern long ptrace (enum __ptrace_request, ...);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
-else
 
-for gdb_arg1 in 'int' 'long'; do
- for gdb_arg2 in 'pid_t' 'int' 'long'; do
-  for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
-   for gdb_arg4 in 'int' 'long' 'void *'; do
+# Provide a safe default value.
+gdb_cv_func_ptrace_proto='int,int,int,long,long'
+
+for gdb_ret in 'int' 'long'; do
+ for gdb_arg1 in 'int' 'long'; do
+  for gdb_arg2 in 'pid_t' 'int' 'long'; do
+   for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
+    for gdb_arg4 in 'int' 'long' 'void *'; do
      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C $gdb_ret
+  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
+
 int
 main ()
 {
 
-extern $gdb_cv_func_ptrace_ret
-  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
-
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
-    break 4;
+  gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
+       break 5;
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-    for gdb_arg5 in 'int *' 'int' 'long'; do
-     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+     for gdb_arg5 in 'int *' 'int' 'long'; do
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 $gdb_ptrace_headers
+EXTERN_C $gdb_ret
+  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
+
 int
 main ()
 {
 
-extern $gdb_cv_func_ptrace_ret
-  ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
-
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_cxx_try_compile "$LINENO"; then :
-
-gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
-    break 5;
+  gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
+        break 6;
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+     done
     done
    done
   done
  done
 done
-# Provide a safe default value.
-: ${gdb_cv_func_ptrace_args='int,int,long,long'}
+
 
 fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_args" >&5
-$as_echo "$gdb_cv_func_ptrace_args" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gdb_cv_func_ptrace_proto" >&5
+$as_echo "$gdb_cv_func_ptrace_proto" >&6; }
+
 ac_save_IFS=$IFS; IFS=','
-set dummy `echo "$gdb_cv_func_ptrace_args" | sed 's/\*/\*/g'`
+set dummy `echo "$gdb_cv_func_ptrace_proto" | sed 's/\*/\*/g'`
 IFS=$ac_save_IFS
 shift
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG1 $1
+#define PTRACE_TYPE_RET $1
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PTRACE_TYPE_ARG1 $2
 _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG3 $3
+#define PTRACE_TYPE_ARG3 $4
 _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG4 $4
+#define PTRACE_TYPE_ARG4 $5
 _ACEOF
 
-if test -n "$5"; then
+if test -n "$6"; then
 
 cat >>confdefs.h <<_ACEOF
-#define PTRACE_TYPE_ARG5 $5
+#define PTRACE_TYPE_ARG5 $6
 _ACEOF
 
 fi
diff --git a/gdb/ptrace.m4 b/gdb/ptrace.m4
index ca2b7c63d96..925c08b9a0e 100644
--- a/gdb/ptrace.m4
+++ b/gdb/ptrace.m4
@@ -36,65 +36,73 @@ gdb_ptrace_headers='
 #if HAVE_UNISTD_H
 # include <unistd.h>
 #endif
+
+#ifdef __cplusplus
+#  define EXTERN_C extern "C"
+#else
+#  define EXTERN_C extern
+#endif
 '
 # There is no point in checking if we don't have a prototype.
 AC_CHECK_DECLS(ptrace, [], [
-  : ${gdb_cv_func_ptrace_ret='int'}
-  : ${gdb_cv_func_ptrace_args='int,int,long,long'}
+  : ${gdb_cv_func_ptrace_proto='int,int,int,long,long'}
 ], $gdb_ptrace_headers)
-# Check return type.  Varargs (used on GNU/Linux) conflict with the
-# empty argument list, so check for that explicitly.
-AC_CACHE_CHECK([return type of ptrace], gdb_cv_func_ptrace_ret,
-  AC_TRY_COMPILE($gdb_ptrace_headers,
-    [extern long ptrace (enum __ptrace_request, ...);],
-    gdb_cv_func_ptrace_ret='long',
-    AC_TRY_COMPILE($gdb_ptrace_headers,
-      [extern int ptrace ();],
-      gdb_cv_func_ptrace_ret='int',
-      gdb_cv_func_ptrace_ret='long')))
-AC_DEFINE_UNQUOTED(PTRACE_TYPE_RET, $gdb_cv_func_ptrace_ret,
-  [Define as the return type of ptrace.])
-# Check argument types.
-AC_CACHE_CHECK([types of arguments for ptrace], gdb_cv_func_ptrace_args, [
-  AC_TRY_COMPILE($gdb_ptrace_headers,
-    [extern long ptrace (enum __ptrace_request, ...);],
-    [gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'],[
-for gdb_arg1 in 'int' 'long'; do
- for gdb_arg2 in 'pid_t' 'int' 'long'; do
-  for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
-   for gdb_arg4 in 'int' 'long' 'void *'; do
-     AC_TRY_COMPILE($gdb_ptrace_headers, [
-extern $gdb_cv_func_ptrace_ret
+
+# GNU/Linux uses a varargs prototype, so check for that explicitly.
+AC_CACHE_CHECK([return type of ptrace], gdb_cv_func_ptrace_proto, [
+  AC_TRY_COMPILE($gdb_ptrace_headers [
+EXTERN_C long ptrace (enum __ptrace_request, ...);
+      ],,
+    [gdb_cv_func_ptrace_proto='long,enum __ptrace_request,int,long,long'])
+])
+
+# Test all possible return and argument types combinations.
+AC_CACHE_CHECK([types of arguments for ptrace], gdb_cv_func_ptrace_proto, [
+
+# Provide a safe default value.
+gdb_cv_func_ptrace_proto='int,int,int,long,long'
+
+for gdb_ret in 'int' 'long'; do
+ for gdb_arg1 in 'int' 'long'; do
+  for gdb_arg2 in 'pid_t' 'int' 'long'; do
+   for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
+    for gdb_arg4 in 'int' 'long' 'void *'; do
+     AC_TRY_COMPILE($gdb_ptrace_headers [
+EXTERN_C $gdb_ret
   ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4);
-], [gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
-    break 4;])
-    for gdb_arg5 in 'int *' 'int' 'long'; do
-     AC_TRY_COMPILE($gdb_ptrace_headers, [
-extern $gdb_cv_func_ptrace_ret
+], [],
+      [gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
+       break 5;])
+     for gdb_arg5 in 'int *' 'int' 'long'; do
+      AC_TRY_COMPILE($gdb_ptrace_headers [
+EXTERN_C $gdb_ret
   ptrace ($gdb_arg1, $gdb_arg2, $gdb_arg3, $gdb_arg4, $gdb_arg5);
-], [
-gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
-    break 5;])
+],,
+       [gdb_cv_func_ptrace_proto="$gdb_ret,$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
+        break 6;])
+     done
     done
    done
   done
  done
 done
-# Provide a safe default value.
-: ${gdb_cv_func_ptrace_args='int,int,long,long'}
-])])
+
+])
+
 ac_save_IFS=$IFS; IFS=','
-set dummy `echo "$gdb_cv_func_ptrace_args" | sed 's/\*/\*/g'`
+set dummy `echo "$gdb_cv_func_ptrace_proto" | sed 's/\*/\*/g'`
 IFS=$ac_save_IFS
 shift
-AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG1, $[1],
+AC_DEFINE_UNQUOTED(PTRACE_TYPE_RET, $[1],
+  [Define as the return type of ptrace.])
+AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG1, $[2],
   [Define to the type of arg 1 for ptrace.])
-AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG3, $[3],
+AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG3, $[4],
   [Define to the type of arg 3 for ptrace.])
-AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG4, $[4],
+AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG4, $[5],
   [Define to the type of arg 4 for ptrace.])
-if test -n "$[5]"; then
-  AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG5, $[5],
+if test -n "$[6]"; then
+  AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG5, $[6],
     [Define to the type of arg 5 for ptrace.])
 fi