Fixed check for flexible array members used in Objective-C instance variables

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@170412 138bc75d-0d04-0410-961f-82ee72b054a4

2 files changed, 102 insertions, 24 deletions

diff --git a/gcc/objc/ChangeLog b/gcc/objc/ChangeLog
index a7b21c322c0..9c466aa8771 100644
--- a/gcc/objc/ChangeLog
+++ b/gcc/objc/ChangeLog
@@ -1,13 +1,22 @@
+2011-02-22  Nicola Pero  <nicola.pero@meta-innovation.com>
+
+	PR objc/47832
+	* objc-act.c (flexible_array_type_p): New.
+	(add_instance_variable): Produce an error if an instance variable
+	uses flexible array members.
+	(encode_array): Do not emit an error if encoding a flexible array
+	type while generating instance variables.
+
 2011-02-21  Mike Stump  <mikestump@comcast.net>
 
 	* Make-lang.in (check_objc_parallelize): Refine for 4 processor machines.
 
 2011-02-20  Nicola Pero  <nicola.pero@meta-innovation.com>
 
-       * objc-gnu-runtime-abi-01.c (TARGET_64BIT): Removed.  Removed
-       usage of padding fields.  Do not include tm.h.
-       * objc-act.c (objc_write_global_declaration): Set input_location
-       to BUILTINS_LOCATION while generating runtime metadata.
+	* objc-gnu-runtime-abi-01.c (TARGET_64BIT): Removed.  Removed
+	usage of padding fields.  Do not include tm.h.
+	* objc-act.c (objc_write_global_declaration): Set input_location
+	to BUILTINS_LOCATION while generating runtime metadata.
 
 2011-01-20  Nicola Pero  <nicola.pero@meta-innovation.com>
 
diff --git a/gcc/objc/objc-act.c b/gcc/objc/objc-act.c
index a91c7083173..85ef8fb0526 100644
--- a/gcc/objc/objc-act.c
+++ b/gcc/objc/objc-act.c
@@ -5925,6 +5925,58 @@ add_category (tree klass, tree category)
     }
 }
 
+#ifndef OBJCPLUS
+/* A flexible array member is a C99 extension where you can use
+   "type[]" at the end of a struct to mean a variable-length array.
+
+   In Objective-C, instance variables are fundamentally members of a
+   struct, but the struct can always be extended by subclassing; hence
+   we need to detect and forbid all instance variables declared using
+   flexible array members.
+
+   No check for this is needed in Objective-C++, since C++ does not
+   have flexible array members.  */
+
+/* Determine whether TYPE is a structure with a flexible array member,
+   a union containing such a structure (possibly recursively) or an
+   array of such structures or unions.  These are all invalid as
+   instance variable.  */
+static bool
+flexible_array_type_p (tree type)
+{
+  tree x;
+  switch (TREE_CODE (type))
+    {
+    case RECORD_TYPE:
+      x = TYPE_FIELDS (type);
+      if (x == NULL_TREE)
+	return false;
+      while (DECL_CHAIN (x) != NULL_TREE)
+	x = DECL_CHAIN (x);
+      if (TREE_CODE (TREE_TYPE (x)) == ARRAY_TYPE
+	  && TYPE_SIZE (TREE_TYPE (x)) == NULL_TREE
+	  && TYPE_DOMAIN (TREE_TYPE (x)) != NULL_TREE
+	  && TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (x))) == NULL_TREE)
+	return true;
+      return false;
+    case UNION_TYPE:
+      for (x = TYPE_FIELDS (type); x != NULL_TREE; x = DECL_CHAIN (x))
+	{
+	  if (flexible_array_type_p (TREE_TYPE (x)))
+	    return true;
+	}
+      return false;
+    /* Note that we also check for arrays of something that uses a flexible array member.  */
+    case ARRAY_TYPE:
+      if (flexible_array_type_p (TREE_TYPE (type)))
+	return true;
+      return false;
+    default:
+    return false;
+  }
+}
+#endif
+
 /* Called after parsing each instance variable declaration. Necessary to
    preserve typedefs and implement public/private...
 
@@ -5958,6 +6010,27 @@ add_instance_variable (tree klass, objc_ivar_visibility_kind visibility,
       return klass;
     }
 
+#ifndef OBJCPLUS
+  /* Also, in C reject a struct with a flexible array member.  Ie,
+
+       struct A { int x; int[] y; };
+
+       @interface X
+       {
+         struct A instance_variable;
+       }
+       @end
+
+       is not valid because if the class is subclassed, we wouldn't be able
+       to calculate the offset of the next instance variable.  */
+  if (flexible_array_type_p (field_type))
+    {
+      error ("instance variable %qs uses flexible array member", ivar_name);
+      /* Return class as is without adding this ivar.  */
+      return klass;      
+    }
+#endif
+
 #ifdef OBJCPLUS
   /* Check if the ivar being added has a non-POD C++ type.   If so, we will
      need to either (1) warn the user about it or (2) generate suitable
@@ -9926,27 +9999,23 @@ encode_array (tree type, int curtype, int format)
   if (an_int_cst == NULL)
     {
       /* We are trying to encode an incomplete array.  An incomplete
-	 array is forbidden as part of an instance variable.  */
-      if (generating_instance_variables)
-	{
-	  /* TODO: Detect this error earlier.  */
-	  error ("instance variable has unknown size");
-	  return;
-	}
+	 array is forbidden as part of an instance variable; but it
+	 may occur if the instance variable is a pointer to such an
+	 array.  */
 
-      /* So the only case in which an incomplete array could occur is
-	 if we are encoding the arguments or return value of a method.
-	 In that case, an incomplete array argument or return value
-	 (eg, -(void)display: (char[])string) is treated like a
-	 pointer because that is how the compiler does the function
-	 call.  A special, more complicated case, is when the
-	 incomplete array is the last member of a struct (eg, if we
-	 are encoding "struct { unsigned long int a;double b[];}"),
-	 which is again part of a method argument/return value.  In
-	 that case, we really need to communicate to the runtime that
-	 there is an incomplete array (not a pointer!) there.  So, we
-	 detect that special case and encode it as a zero-length
-	 array.
+      /* So the only case in which an incomplete array could occur
+	 (without being pointed to) is if we are encoding the
+	 arguments or return value of a method.  In that case, an
+	 incomplete array argument or return value (eg,
+	 -(void)display: (char[])string) is treated like a pointer
+	 because that is how the compiler does the function call.  A
+	 special, more complicated case, is when the incomplete array
+	 is the last member of a struct (eg, if we are encoding
+	 "struct { unsigned long int a;double b[];}"), which is again
+	 part of a method argument/return value.  In that case, we
+	 really need to communicate to the runtime that there is an
+	 incomplete array (not a pointer!) there.  So, we detect that
+	 special case and encode it as a zero-length array.
 
 	 Try to detect that we are part of a struct.  We do this by
 	 searching for '=' in the type encoding for the current type.