THRIFT-5626 Parser should not confuse data types and field names

Patch: Jens Geyer

2 files changed, 216 insertions, 57 deletions

diff --git a/compiler/cpp/src/thrift/thriftl.ll b/compiler/cpp/src/thrift/thriftl.ll
index 1cc3d2388..45c59260d 100644
--- a/compiler/cpp/src/thrift/thriftl.ll
+++ b/compiler/cpp/src/thrift/thriftl.ll
@@ -209,29 +209,16 @@ literal_begin (['\"])
 "true"               { yylval.iconst=1; return tok_int_constant; }
 
 "namespace"          { return tok_namespace;            }
-"cpp_namespace"      { error_unsupported_namespace_decl("cpp"); /* do nothing */ }
 "cpp_include"        { return tok_cpp_include;          }
 "cpp_type"           { return tok_cpp_type;             }
-"java_package"       { error_unsupported_namespace_decl("java_package", "java"); /* do nothing */ }
-"delphi_namespace"   { error_unsupported_namespace_decl("delphi"); /* do nothing */ }
-"php_namespace"      { error_unsupported_namespace_decl("php"); /* do nothing */ }
-"py_module"          { error_unsupported_namespace_decl("py_module", "py"); /* do nothing */ }
-"perl_package"       { error_unsupported_namespace_decl("perl_package", "perl"); /* do nothing */ }
-"ruby_namespace"     { error_unsupported_namespace_decl("ruby"); /* do nothing */ }
-"smalltalk_category" { error_unsupported_namespace_decl("smalltalk_category", "st"); /* do nothing */ }
-"smalltalk_prefix"   { error_unsupported_namespace_decl("smalltalk_prefix", "st"); /* do nothing */ }
 "xsd_all"            { return tok_xsd_all;              }
 "xsd_optional"       { return tok_xsd_optional;         }
 "xsd_nillable"       { return tok_xsd_nillable;         }
-"xsd_namespace"      { error_unsupported_namespace_decl("xsd"); /* do nothing */ }
 "xsd_attrs"          { return tok_xsd_attrs;            }
 "include"            { return tok_include;              }
 "void"               { return tok_void;                 }
 "bool"               { return tok_bool;                 }
-"byte"               {
-  emit_byte_type_warning();
-  return tok_i8;
-}
+"byte"               { emit_byte_type_warning(); return tok_byte; }
 "i8"                 { return tok_i8;                   }
 "i16"                { return tok_i16;                  }
 "i32"                { return tok_i32;                  }
@@ -240,12 +227,6 @@ literal_begin (['\"])
 "string"             { return tok_string;               }
 "binary"             { return tok_binary;               }
 "uuid"               { return tok_uuid;                 }
-"slist" {
-  error_no_longer_supported("slist","string");
-}
-"senum" {
-  error_no_longer_supported("senum","string");
-}
 "map"                { return tok_map;                  }
 "list"               { return tok_list;                 }
 "set"                { return tok_set;                  }
@@ -263,7 +244,7 @@ literal_begin (['\"])
 "optional"           { return tok_optional;             }
 "async" {
   pwarning(0, "\"async\" is deprecated.  It is called \"oneway\" now.\n");
-  return tok_oneway;
+  return tok_async;
 }
 "&"                  { return tok_reference;            }
 
diff --git a/compiler/cpp/src/thrift/thrifty.yy b/compiler/cpp/src/thrift/thrifty.yy
index 40c2a9373..82b2be57e 100644
--- a/compiler/cpp/src/thrift/thrifty.yy
+++ b/compiler/cpp/src/thrift/thrifty.yy
@@ -97,6 +97,7 @@ const int struct_is_union = 1;
   t_function*    tfunction;
   t_field*       tfield;
   char*          dtext;
+  char*          keyword;
   t_field::e_req ereq;
   t_annotation*  tannot;
   t_field_id     tfieldid;
@@ -118,56 +119,58 @@ const int struct_is_union = 1;
 /**
  * Header keywords
  */
-%token tok_include
-%token tok_namespace
-%token tok_cpp_include
-%token tok_cpp_type
-%token tok_xsd_all
-%token tok_xsd_optional
-%token tok_xsd_nillable
-%token tok_xsd_attrs
+%token<keyword> tok_include
+%token<keyword> tok_namespace
+%token<keyword> tok_cpp_include
+%token<keyword> tok_cpp_type
+%token<keyword> tok_xsd_all
+%token<keyword> tok_xsd_optional
+%token<keyword> tok_xsd_nillable
+%token<keyword> tok_xsd_attrs
 
 /**
  * Base datatype keywords
  */
-%token tok_void
-%token tok_bool
-%token tok_string
-%token tok_binary
-%token tok_uuid
-%token tok_i8
-%token tok_i16
-%token tok_i32
-%token tok_i64
-%token tok_double
+%token<keyword> tok_void
+%token<keyword> tok_bool
+%token<keyword> tok_string
+%token<keyword> tok_binary
+%token<keyword> tok_uuid
+%token<keyword> tok_byte
+%token<keyword> tok_i8
+%token<keyword> tok_i16
+%token<keyword> tok_i32
+%token<keyword> tok_i64
+%token<keyword> tok_double
 
 /**
  * Complex type keywords
  */
-%token tok_map
-%token tok_list
-%token tok_set
+%token<keyword> tok_map
+%token<keyword> tok_list
+%token<keyword> tok_set
 
 /**
  * Function modifiers
  */
-%token tok_oneway
+%token<keyword> tok_oneway
+%token<keyword> tok_async
 
 /**
  * Thrift language keywords
  */
-%token tok_typedef
-%token tok_struct
-%token tok_xception
-%token tok_throws
-%token tok_extends
-%token tok_service
-%token tok_enum
-%token tok_const
-%token tok_required
-%token tok_optional
-%token tok_union
-%token tok_reference
+%token<keyword> tok_typedef
+%token<keyword> tok_struct
+%token<keyword> tok_xception
+%token<keyword> tok_throws
+%token<keyword> tok_extends
+%token<keyword> tok_service
+%token<keyword> tok_enum
+%token<keyword> tok_const
+%token<keyword> tok_required
+%token<keyword> tok_optional
+%token<keyword> tok_union
+%token<keyword> tok_reference
 
 /**
  * Grammar nodes
@@ -193,6 +196,7 @@ const int struct_is_union = 1;
 
 %type<tfield>    Field
 %type<tfieldid>  FieldIdentifier
+%type<id>        FieldName
 %type<ereq>      FieldRequiredness
 %type<ttype>     FieldType
 %type<tconstv>   FieldValue
@@ -771,6 +775,10 @@ Oneway:
     {
       $$ = true;
     }
+|  tok_async  // deprecated
+    {
+      $$ = true;
+    }
 |
     {
       $$ = false;
@@ -809,9 +817,9 @@ FieldList:
     }
 
 Field:
-  CaptureDocText FieldIdentifier FieldRequiredness FieldType FieldReference tok_identifier FieldValue XsdOptional XsdNillable XsdAttributes TypeAnnotations CommaOrSemicolonOptional
+  CaptureDocText FieldIdentifier FieldRequiredness FieldType FieldReference FieldName FieldValue XsdOptional XsdNillable XsdAttributes TypeAnnotations CommaOrSemicolonOptional
     {
-      pdebug("tok_int_constant : Field -> FieldType tok_identifier");
+      pdebug("tok_int_constant : Field -> FieldType FieldName");
       if ($2.auto_assigned) {
         pwarning(1, "No field key specified for %s, resulting protocol may have conflicts or not be backwards compatible!\n", $6);
         if (g_strict >= 192) {
@@ -842,6 +850,171 @@ Field:
       }
     }
 
+FieldName:  // identifiers and everything that could be one if it would not be identified as a different token already and excluding the "xsd*" keywords to follow a FieldName
+  tok_identifier
+    {
+      pdebug("FieldName -> tok_identifier");
+      $$ = $1;
+    }
+| tok_namespace
+    {
+      pdebug("FieldName -> tok_namespace");
+      $$ = strdup("namespace");
+    }
+| tok_cpp_include
+    {
+      pdebug("FieldName -> tok_cpp_include");
+      $$ = strdup("cpp_include");
+    }
+/* see THRIFT-5627 "More consistent syntax for cpp_type" -> activate when this issue is resolved
+| tok_cpp_type
+    {
+      pdebug("FieldName -> tok_cpp_type");
+      $$ = $strdup("cpp_type");
+    }
+*/
+| tok_include
+    {
+      pdebug("FieldName -> tok_include");
+      $$ = strdup("include");
+    }
+| tok_void
+    {
+      pdebug("FieldName -> tok_void");
+      $$ = strdup("void");
+    }
+| tok_bool
+    {
+      pdebug("FieldName -> tok_bool");
+      $$ = strdup("bool");
+    }
+| tok_byte
+    {
+      pdebug("FieldName -> tok_byte");
+      $$ = strdup("byte");
+    }
+| tok_i8
+    {
+      pdebug("FieldName -> tok_i8");
+      $$ = strdup("i8");
+    }
+| tok_i16
+    {
+      pdebug("FieldName -> tok_i16");
+      $$ = strdup("i16");
+    }
+| tok_i32
+    {
+      pdebug("FieldName -> tok_i32");
+      $$ = strdup("i32");
+    }
+| tok_i64
+    {
+      pdebug("FieldName -> tok_i64");
+      $$ = strdup("i64");
+    }
+| tok_double
+    {
+      pdebug("FieldName -> tok_double");
+      $$ = strdup("double");
+    }
+| tok_string
+    {
+      pdebug("FieldName -> tok_string");
+      $$ = strdup("string");
+    }
+| tok_binary
+    {
+      pdebug("FieldName -> tok_binary");
+      $$ = strdup("binary");
+    }
+| tok_uuid
+    {
+      pdebug("FieldName -> tok_uuid");
+      $$ = strdup("uuid");
+    }
+| tok_map
+    {
+      pdebug("FieldName -> tok_map");
+      $$ = strdup("map");
+    }
+| tok_list
+    {
+      pdebug("FieldName -> tok_list");
+      $$ = strdup("list");
+    }
+| tok_set
+    {
+      pdebug("FieldName -> tok_set");
+      $$ = strdup("set");
+    }
+| tok_oneway
+    {
+      pdebug("FieldName -> tok_oneway");
+      $$ = strdup("oneway");
+    }
+| tok_async
+    {
+      pdebug("FieldName -> tok_async");
+      $$ = strdup("async");
+    }
+| tok_typedef
+    {
+      pdebug("FieldName -> tok_typedef");
+      $$ = strdup("typedef");
+    }
+| tok_struct
+    {
+      pdebug("FieldName -> tok_struct");
+      $$ = strdup("struct");
+    }
+| tok_union
+    {
+      pdebug("FieldName -> tok_union");
+      $$ = strdup("union");
+    }
+| tok_xception
+    {
+      pdebug("FieldName -> tok_xception");
+      $$ = strdup("exception");
+    }
+| tok_extends
+    {
+      pdebug("FieldName -> tok_extends");
+      $$ = strdup("extends");
+    }
+| tok_throws
+    {
+      pdebug("FieldName -> tok_throws");
+      $$ = strdup("throws");
+    }
+| tok_service
+    {
+      pdebug("FieldName -> tok_service");
+      $$ = strdup("service");
+    }
+| tok_enum
+    {
+      pdebug("FieldName -> tok_enum");
+      $$ = strdup("enum");
+    }
+| tok_const
+    {
+      pdebug("FieldName -> tok_const");
+      $$ = strdup("const");
+    }
+| tok_required
+    {
+      pdebug("FieldName -> tok_required");
+      $$ = strdup("required");
+    }
+| tok_optional
+    {
+      pdebug("FieldName -> tok_optional");
+      $$ = strdup("optional");
+    }
+  
+  
 FieldIdentifier:
   tok_int_constant ':'
     {
@@ -1013,6 +1186,11 @@ SimpleBaseType:
       pdebug("BaseType -> tok_bool");
       $$ = g_type_bool;
     }
+| tok_byte
+    {
+      pdebug("BaseType -> tok_byte");
+      $$ = g_type_i8;  // byte is signed in Thrift, just an alias for i8
+    }
 | tok_i8
     {
       pdebug("BaseType -> tok_i8");