diff options
| -rw-r--r-- | NEWS | 6 | ||||
| -rw-r--r-- | data/lalr1.java | 262 | ||||
| -rw-r--r-- | doc/bison.texi | 70 | ||||
| -rw-r--r-- | tests/javapush.at | 850 | ||||
| -rw-r--r-- | tests/local.mk | 1 | ||||
| -rw-r--r-- | tests/testsuite.at | 7 |
6 files changed, 1142 insertions, 54 deletions
@@ -519,11 +519,13 @@ GNU Bison NEWS ** Java skeleton improvements - Contributed by Paolo Bonzini. - The constants for token names were moved to the Lexer interface. Also, it is possible to add code to the parser's constructors using "%code init" and "%define init_throws". + Contributed by Paolo Bonzini. + + The Java skeleton now supports push parsing. + Contributed by Dennis Heimbigner. ** C++ skeletons improvements diff --git a/data/lalr1.java b/data/lalr1.java index 5c0235f0..d226ce3c 100644 --- a/data/lalr1.java +++ b/data/lalr1.java @@ -20,7 +20,8 @@ m4_include(b4_pkgdatadir/[java.m4]) b4_defines_if([b4_fatal([%s: %%defines does not make sense in Java], [b4_skeleton])]) -# We don't depend on %debug in Java, but pacify warnings about non-used flags. +# We do not depend on %debug in Java, but pacify warnings about +# non-used flags. b4_parse_trace_if([0], [0]) m4_define([b4_symbol_no_destructor_assert], @@ -30,6 +31,57 @@ m4_define([b4_symbol_no_destructor_assert], [b4_symbol_action_location([$1], [destructor])])])]) b4_symbol_foreach([b4_symbol_no_destructor_assert]) +# Setup some macros for api.push-pull. +b4_percent_define_default([[api.push-pull]], [[pull]]) +b4_percent_define_check_values([[[[api.push-pull]], + [[pull]], [[push]], [[both]]]]) + +# Define m4 conditional macros that encode the value +# of the api.push-pull flag. +b4_define_flag_if([pull]) m4_define([b4_pull_flag], [[1]]) +b4_define_flag_if([push]) m4_define([b4_push_flag], [[1]]) +m4_case(b4_percent_define_get([[api.push-pull]]), + [pull], [m4_define([b4_push_flag], [[0]])], + [push], [m4_define([b4_pull_flag], [[0]])]) + +# Define a macro to be true when api.push-pull has the value "both". +m4_define([b4_both_if],[b4_push_if([b4_pull_if([$1],[$2])],[$2])]) + +# Handle BISON_USE_PUSH_FOR_PULL for the test suite. So that push parsing +# tests function as written, do not let BISON_USE_PUSH_FOR_PULL modify the +# behavior of Bison at all when push parsing is already requested. +b4_define_flag_if([use_push_for_pull]) +b4_use_push_for_pull_if([ + b4_push_if([m4_define([b4_use_push_for_pull_flag], [[0]])], + [m4_define([b4_push_flag], [[1]])])]) + +# Define a macro to encapsulate the parse state variables. +# This allows them to be defined either in parse() when doing +# pull parsing, or as class instance variable when doing push parsing. +m4_define([b4_define_state],[[ + /* Lookahead and lookahead in internal form. */ + int yychar = yyempty_; + int yytoken = 0; + + /* State. */ + int yyn = 0; + int yylen = 0; + int yystate = 0; + YYStack yystack = new YYStack (); + int label = YYNEWSTATE; + + /* Error handling. */ + int yynerrs_ = 0; + ]b4_locations_if([/* The location where the error started. */ + b4_location_type yyerrloc = null; + + /* Location. */ + b4_location_type yylloc = new b4_location_type (null, null);])[ + + /* Semantic value of the lookahead. */ + ]b4_yystype[ yylval = null; +]]) + b4_output_begin([b4_parser_file_name]) b4_copyright([Skeleton implementation for Bison LALR(1) parsers in Java], [2007-2013]) @@ -363,6 +415,12 @@ b4_lexer_if([[ */ public static final int YYABORT = 1; +]b4_push_if([ + /** + * Returned by a Bison action in order to request a new token. + */ + public static final int YYMORE = 4;])[ + /** * Returned by a Bison action in order to start error recovery without * printing an error message. @@ -379,9 +437,12 @@ b4_lexer_if([[ private static final int YYREDUCE = 6; private static final int YYERRLAB1 = 7; private static final int YYRETURN = 8; +]b4_push_if([[ private static final int YYGETTOKEN = 9; /* Signify that a new token is expected when doing push-parsing. */]])[ private int yyerrstatus_ = 0; +]b4_push_if([dnl +b4_define_state])[ /** * Return whether error recovery is being done. In this state, the parser * reads token until it reaches a known state, and then restarts normal @@ -486,6 +547,7 @@ b4_lexer_if([[ + (yyvaluep == null ? "(null)" : yyvaluep.toString ()) + ")"); } +]b4_push_if([],[[ /** * Parse input from the scanner that was specified at object construction * time. Return whether the end of the input was reached successfully. @@ -493,46 +555,53 @@ b4_lexer_if([[ * @@return <tt>true</tt> if the parsing succeeds. Note that this does not * imply that there were no syntax errors. */ - public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ + public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[ +]b4_push_if([ + /** + * Push Parse input from external lexer + * + * @@param yylextoken current token + * @@param yylexval current lval +]b4_locations_if([ * @@param yylexloc current position])[ + * + * @@return <tt>YYACCEPT, YYABORT, YYMORE</tt> + */ + public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc])) + b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[ { - /// Lookahead and lookahead in internal form. - int yychar = yyempty_; - int yytoken = 0; - - /* State. */ - int yyn = 0; - int yylen = 0; - int yystate = 0; - - YYStack yystack = new YYStack (); - - /* Error handling. */ - int yynerrs_ = 0; - ]b4_locations_if([/// The location where the error started. - ]b4_location_type[ yyerrloc = null; - - /// ]b4_location_type[ of the lookahead. - ]b4_location_type[ yylloc = new ]b4_location_type[ (null, null); - - /// @@$. - ]b4_location_type[ yyloc;]) - - /// Semantic value of the lookahead. - b4_yystype[ yylval = null; - + ]b4_locations_if([/* @@$. */ + b4_location_type yyloc;])[ +]b4_push_if([],[[ +]b4_define_state[ yycdebug ("Starting parse\n"); yyerrstatus_ = 0; + /* Initialize the stack. */ + yystack.push (yystate, yylval ]b4_locations_if([, yylloc])[); ]m4_ifdef([b4_initial_action], [ b4_dollar_pushdef([yylval], [], [yylloc])dnl /* User initialization code. */ b4_user_initial_action -b4_dollar_popdef])[]dnl - - [ /* Initialize the stack. */ - yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); - - int label = YYNEWSTATE; +b4_dollar_popdef[]dnl +])[ +]])[ +]b4_push_if([[ + if (!this.push_parse_initialized) + { + push_parse_initialize (); +]m4_ifdef([b4_initial_action], [ +b4_dollar_pushdef([yylval], [], [yylloc])dnl + /* User initialization code. */ + b4_user_initial_action +b4_dollar_popdef[]dnl +])[ + yycdebug ("Starting parse\n"); + yyerrstatus_ = 0; + } else + label = YYGETTOKEN; + + boolean push_token_consumed = true; +]])[ for (;;) switch (label) { @@ -545,7 +614,8 @@ b4_dollar_popdef])[]dnl /* Accept? */ if (yystate == yyfinal_) - return true; + ]b4_push_if([{label = YYACCEPT; break;}], + [return true;])[ /* Take a decision. First try without lookahead. */ yyn = yypact_[yystate]; @@ -554,16 +624,27 @@ b4_dollar_popdef])[]dnl label = YYDEFAULT; break; } +]b4_push_if([ /* Fall Through */ + case YYGETTOKEN:])[ /* Read a lookahead token. */ if (yychar == yyempty_) { +]b4_push_if([[ + if (!push_token_consumed) + return YYMORE; + yycdebug ("Reading a token: "); + yychar = yylextoken; + yylval = yylexval;]b4_locations_if([ + yylloc = yylexloc;])[ + push_token_consumed = false;]])[ +]b4_push_if([],[[ yycdebug ("Reading a token: "); - yychar = yylexer.yylex ();] - b4_locations_if([[ - yylloc = new ]b4_location_type[(yylexer.getStartPos (), - yylexer.getEndPos ());]]) - yylval = yylexer.getLVal ();[ + yychar = yylexer.yylex (); + yylval = yylexer.getLVal ();]b4_locations_if([ + yylloc = new b4_location_type (yylexer.getStartPos (), + yylexer.getEndPos ());])[ +]])[ } /* Convert token to internal form. */ @@ -660,10 +741,10 @@ b4_dollar_popdef])[]dnl { /* Return failure if at end of input. */ if (yychar == Lexer.EOF) - return false; + ]b4_push_if([{label = YYABORT; break;}],[return false;])[ } else - yychar = yyempty_; + yychar = yyempty_; } /* Else will try to reuse lookahead token after shifting the error @@ -671,9 +752,9 @@ b4_dollar_popdef])[]dnl label = YYERRLAB1; break; - /*---------------------------------------------------. + /*-------------------------------------------------. | errorlab -- error raised explicitly by YYERROR. | - `---------------------------------------------------*/ + `-------------------------------------------------*/ case YYERROR: ]b4_locations_if([yyerrloc = yystack.locationAt (yylen - 1);])[ @@ -705,9 +786,10 @@ b4_dollar_popdef])[]dnl } } - /* Pop the current state because it cannot handle the error token. */ + /* Pop the current state because it cannot handle the + * error token. */ if (yystack.height == 0) - return false; + ]b4_push_if([{label = YYABORT; break;}],[return false;])[ ]b4_locations_if([yyerrloc = yystack.locationAt (0);])[ yystack.pop (); @@ -716,7 +798,11 @@ b4_dollar_popdef])[]dnl yystack.print (yyDebugStream); } - ]b4_locations_if([ + if (label == YYABORT) + /* Leave the switch. */ + break; + +]b4_locations_if([ /* Muck with the stack to setup for yylloc. */ yystack.push (0, null, yylloc); yystack.push (0, null, yyerrloc); @@ -734,13 +820,92 @@ b4_dollar_popdef])[]dnl /* Accept. */ case YYACCEPT: - return true; + ]b4_push_if([this.push_parse_initialized = false; return YYACCEPT;], + [return true;])[ /* Abort. */ case YYABORT: - return false; + ]b4_push_if([this.push_parse_initialized = false; return YYABORT;], + [return false;])[ } +} +]b4_push_if([[ + boolean push_parse_initialized = false; + + /** + * (Re-)Initialize the state of the push parser. + */ + public void push_parse_initialize() + { + /* Lookahead and lookahead in internal form. */ + this.yychar = yyempty_; + this.yytoken = 0; + + /* State. */ + this.yyn = 0; + this.yylen = 0; + this.yystate = 0; + this.yystack = new YYStack (); + this.label = YYNEWSTATE; + + /* Error handling. */ + this.yynerrs_ = 0; + ]b4_locations_if([/* The location where the error started. */ + this.yyerrloc = null; + this.yylloc = new b4_location_type (null, null);])[ + + /* Semantic value of the lookahead. */ + this.yylval = null; + + yystack.push (this.yystate, this.yylval]b4_locations_if([, this.yylloc])[); + + this.push_parse_initialized = true; + } +]b4_locations_if([ + /** + * Push parse given input from an external lexer. + * + * @@param yylextoken current token + * @@param yylexval current lval + * @@param yyylexpos current position + * + * @@return <tt>YYACCEPT, YYABORT, YYMORE</tt> + */ + public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos) + b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])]) + { + return push_parse (yylextoken, yylexval, new b4_location_type (yylexpos)); + } +])[]]) + +b4_both_if([[ + /** + * Parse input from the scanner that was specified at object construction + * time. Return whether the end of the input was reached successfully. + * This version of parse () is defined only when api.push-push=both. + * + * @@return <tt>true</tt> if the parsing succeeds. Note that this does not + * imply that there were no syntax errors. + */ + public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ + { + if (yylexer == null) + throw new NullPointerException("Null Lexer"); + int status; + do { + int token = yylexer.yylex(); + ]b4_yystype[ lval = yylexer.getLVal(); +]b4_locations_if([dnl + b4_location_type yyloc = new b4_location_type (yylexer.getStartPos (), + yylexer.getEndPos ());])[ + this.yyerrstatus_ = 0; + ]b4_locations_if([status = push_parse(token,lval,yyloc);],[ + status = push_parse(token,lval);])[ + } while (status == YYMORE); + return (status == YYACCEPT); + } +]])[ // Generate an error message. private String yysyntax_error (int yystate, int tok) @@ -849,6 +1014,7 @@ b4_dollar_popdef])[]dnl ]b4_integral_parser_table_define([rline], [b4_rline], [[YYRLINE[YYN] -- Source line where rule number YYN was defined.]])[ + // Report on the debug stream that the rule yyrule is going to be reduced. private void yy_reduce_print (int yyrule, YYStack yystack) { diff --git a/doc/bison.texi b/doc/bison.texi index 4a4bd600..bd023edf 100644 --- a/doc/bison.texi +++ b/doc/bison.texi @@ -366,6 +366,7 @@ Java Parsers * Java Parser Interface:: Instantiating and running the parser * Java Scanner Interface:: Specifying the scanner for the parser * Java Action Features:: Special features for use in actions +* Java Push Parser Interface:: Instantiating and running the a push parser * Java Differences:: Differences between C/C++ and Java Grammars * Java Declarations Summary:: List of Bison declarations used with Java @@ -11500,6 +11501,7 @@ main (int argc, char *argv[]) * Java Parser Interface:: Instantiating and running the parser * Java Scanner Interface:: Specifying the scanner for the parser * Java Action Features:: Special features for use in actions +* Java Push Parser Interface:: Instantiating and running the a push parser * Java Differences:: Differences between C/C++ and Java Grammars * Java Declarations Summary:: List of Bison declarations used with Java @end menu @@ -11811,7 +11813,6 @@ The return type can be changed using @samp{%define api.value.type @{@var{class-name}@}}. @end deftypemethod - @node Java Action Features @subsection Special Features for Use in Java Actions @@ -11890,6 +11891,73 @@ instance in use. The @code{Location} and @code{Position} parameters are available only if location tracking is active. @end deftypefn +@node Java Push Parser Interface +@subsection Java Push Parser Interface +@c - define push_parse +@findex %define api.push-pull + +(The current push parsing interface is experimental and may evolve. More +user feedback will help to stabilize it.) + +Normally, Bison generates a pull parser for Java. +The following Bison declaration says that you want the parser to be a push +parser (@pxref{%define Summary,,api.push-pull}): + +@example +%define api.push-pull push +@end example + +Most of the discussion about the Java pull Parser Interface, (@pxref{Java +Parser Interface}) applies to the push parser interface as well. + +When generating a push parser, the method @code{push_parse} is created with +the following signature (depending on if locations are enabled). + +@deftypemethod {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}) +@deftypemethodx {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}, {Location} @var{yyloc}) +@deftypemethodx {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}, {Position} @var{yypos}) +@end deftypemethod + +The primary difference with respect to a pull parser is that the parser +method @code{push_parse} is invoked repeatedly to parse each token. This +function is available if either the "%define api.push-pull push" or "%define +api.push-pull both" declaration is used (@pxref{%define +Summary,,api.push-pull}). The @code{Location} and @code{Position} +parameters are available only if location tracking is active. + +The value returned by the @code{push_parse} method is one of the following +four constants: @code{YYABORT}, @code{YYACCEPT}, @code{YYERROR}, or +@code{YYMORE}. This new value, @code{YYMORE}, may be returned if more input +is required to finish parsing the grammar. + +If api.push-pull is declared as @code{both}, then the generated parser class +will also implement the @code{parse} method. This method's body is a loop +that repeatedly invokes the scanner and then passes the values obtained from +the scanner to the @code{push_parse} method. + +There is one additional complication. Technically, the push parser does not +need to know about the scanner (i.e. an object implementing the +@code{YYParser.Lexer} interface), but it does need access to the +@code{yyerror} method. Currently, the @code{yyerror} method is defined in +the @code{YYParser.Lexer} interface. Hence, an implementation of that +interface is still required in order to provide an implementation of +@code{yyerror}. The current approach (and subject to change) is to require +the @code{YYParser} constructor to be given an object implementing the +@code{YYParser.Lexer} interface. This object need only implement the +@code{yyerror} method; the other methods can be stubbed since they will +never be invoked. The simplest way to do this is to add a trivial scanner +implementation to your grammar file using whatever implementation of +@code{yyerror} is desired. The following code sample shows a simple way to +accomplish this. + +@example +%code lexer +@{ + public Object getLVal () @{return null;@} + public int yylex () @{return 0;@} + public void yyerror (String s) @{System.err.println(s);@} +@} +@end example @node Java Differences @subsection Differences between C/C++ and Java Grammars diff --git a/tests/javapush.at b/tests/javapush.at new file mode 100644 index 00000000..65027c31 --- /dev/null +++ b/tests/javapush.at @@ -0,0 +1,850 @@ +# Checking Java Push Parsing. -*- Autotest -*- + +# Copyright (C) 2013 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# The Java push parser tests are intended primarily +# to verify that the sequence of states that the parser +# traverses is the same as a pull parser would traverse. + +################################################## +# Provide a way to generate data with and without push parsing +# so it is possible to capture the output for comparison +# (except the "trivial" tests). +# Use "both" rather than "push" so we can also set it to "pull" to +# get the "experr" data. + +m4_define([PUSHPULLFLAG],[-Dapi.push-pull=both]) + +# Modify the grep tester from java.at +# to ignore status so we can test for zero matching lines +m4_define([AT_CHECK_JAVA_GREP], + [AT_CHECK([grep -c '^$2$' $1], [ignore], [m4_default([$3], [1]) +]) +]) + +################################################## + +AT_BANNER([[Java Push Parsing Tests]]) + +# Define a single copy of the trivial parser grammar. +# This is missing main(), so two versions +# are instantiated with different main() procedures. +m4_define([AT_TRIVIAL_GRAMMAR],[ +%define parser_class_name {YYParser} +%error-verbose + +%code imports { +import java.io.*; +import java.util.*; +} + +%% + +start: 'a' 'b' 'c' ; + +%% +]) + +# Define comon code across to be includede in +# class Main for the trivial parser tests. +m4_define([AT_TRIVIAL_COMMON],[ + static class YYerror implements YYParser.Lexer + { + public Object getLVal() {return null;} + public int yylex () throws java.io.IOException { return 0; } + public void yyerror (String msg) { System.err.println(msg); } + } + + static YYParser parser = null; + static YYerror yyerror = null; + static int teststate = -1; + + static void setup() + throws IOException + { + yyerror = new YYerror(); + parser = new YYParser(yyerror); + parser.setDebugLevel(1); + teststate = -1; + } + + static String[[]] teststatename + = new String[[]]{"YYACCEPT","YYABORT","YYERROR","UNKNOWN","YYMORE"}; + + static void check(int teststate, int expected, String msg) + { + System.err.println("teststate="+teststatename[[teststate]] + +"; expected="+teststatename[[expected]]); + if (teststate == expected) + return; + System.err.println("unexpected state: "+msg); + System.exit(1); + } +]) + +m4_define([AT_TRIVIAL_PARSER],[ + AT_TRIVIAL_GRAMMAR + + public class Main + { + + AT_TRIVIAL_COMMON + + static public void main (String[[]] argv) + throws IOException + { + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYMORE,"push_parse('b', null)"); + teststate = parser.push_parse('c', null); + check(teststate,YYParser.YYMORE,"push_parse('c', null)"); + teststate = parser.push_parse('\0', null); + check(teststate,YYParser.YYACCEPT,"push_parse('\\0', null)"); + + /* Reuse the parser instance and cause a failure */ + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYABORT,"push_parse('b', null)"); + + System.exit(0); + } + +} +]) + +m4_define([AT_TRIVIAL_PARSER_INITIAL_ACTION],[ + AT_TRIVIAL_GRAMMAR + + public class Main + { + + AT_TRIVIAL_COMMON + + static public void main (String[[]] argv) + throws IOException + { + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYMORE,"push_parse('b', null)"); + teststate = parser.push_parse('c', null); + check(teststate,YYParser.YYMORE,"push_parse('c', null)"); + teststate = parser.push_parse('\0', null); + check(teststate,YYParser.YYACCEPT,"push_parse('\\0', null)"); + + System.exit(0); + } + +} +]) + +AT_SETUP([Trivial Push Parser with api.push-pull verification]) +AT_BISON_OPTION_PUSHDEFS +AT_DATA([[input.y]],[[%language "Java" +]AT_TRIVIAL_PARSER[ +]]) +AT_BISON_OPTION_POPDEFS + +# Verify that the proper procedure(s) are generated for each case. +AT_BISON_CHECK([[-Dapi.push-pull=pull -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [1]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [0]) +AT_BISON_CHECK([[-Dapi.push-pull=both -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [1]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [1]) +AT_BISON_CHECK([[-Dapi.push-pull=push -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [0]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [1]) + +AT_JAVA_COMPILE([[Main.java]]) +AT_JAVA_PARSER_CHECK([Main], 0, [], [stderr-nolog]) +AT_CLEANUP + +AT_SETUP([Trivial Push Parser with %initial-action]) +AT_BISON_OPTION_PUSHDEFS +AT_DATA([[input.y]],[[%language "Java" +%initial-action { +System.err.println("Initial action invoked"); +} +]AT_TRIVIAL_PARSER_INITIAL_ACTION[ +]]) +AT_BISON_OPTION_POPDEFS +AT_BISON_CHECK([[-Dapi.push-pull=push -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[System.err.println("Initial action invoked");]]) +AT_JAVA_COMPILE([[Main.java]]) +AT_JAVA_PARSER_CHECK([Main], 0, [], [stderr-nolog]) +# Verify that initial action is called exactly once. +AT_CHECK_JAVA_GREP( + [[stderr]], + [[Initial action invoked]], + [1]) +AT_CLEANUP + +# Define a single copy of the Calculator grammar. +m4_define([AT_CALC_BODY],[ +%code imports { + import java.io.*; +} + +%code { + static StringReader + getinput(String filename) throws IOException + { + StringBuilder buf = new StringBuilder(); + FileReader file = new FileReader(filename); + int c; + while ((c=file.read()) > 0) + buf.append((char)c); + file.close(); + return new StringReader(buf.toString()); + } +} + +/* Bison Declarations */ +%token <Integer> NUM "number" +%type <Integer> exp + +%nonassoc '=' /* comparison */ +%left '-' '+' +%left '*' '/' +%left NEG /* negation--unary minus */ +%right '^' /* exponentiation */ + +/* Grammar follows */ +%% +input: + line +| input line +; + +line: + '\n' +| exp '\n' + {System.out.println("total = "+$[]1);} +| error '\n' +; + +exp: + NUM { $[]$ = $[]1;} +| exp '=' exp + { + if ($[]1.intValue() != $[]3.intValue()) + yyerror (]AT_LOCATION_IF([[@$,]])[ "calc: error: " + $[]1 + " != " + $[]3); + } +| exp '+' exp + { $[]$ = new Integer ($[]1.intValue () + $[]3.intValue ()); } +| exp '-' exp + { $[]$ = new Integer ($[]1.intValue () - $[]3.intValue ()); } +| exp '*' exp + { $[]$ = new Integer ($[]1.intValue () * $[]3.intValue ()); } +| exp '/' exp + { $[]$ = new Integer ($[]1.intValue () / $[]3.intValue ()); } +| '-' exp %prec NEG + { $[]$ = new Integer (-$[]2.intValue ()); } +| exp '^' exp + { $[]$ = new Integer ((int)Math.pow ($[]1.intValue (), + $[]3.intValue ())); } +| '(' exp ')' { $[]$ = $[]2;} +| '(' error ')' { $[]$ = new Integer (1111);} +| '!' { $[]$ = new Integer (0); return YYERROR;} +| '-' error { $[]$ = new Integer (0); return YYERROR;} +; +]) + +# Test that the states transitioned by the push parser are the +# same as for the pull parser. This test is assumed to work +# if it produces the same partial trace of stack states as is +# produced when using pull parsing. The output is verbose, +# but seems essential for verifying push parsing. + +AT_SETUP([Calc parser with api.push-pull both]) +AT_BISON_OPTION_PUSHDEFS + +# Define the calculator input. +# Warning: if you changes the input file +# then the locations test file position numbers +# may be incorrect and you will have +# to modify that file as well. + +AT_DATA([input],[[1 + 2 * 3 = 7 +1 + 2 * -3 = -5 + +-1^2 = -1 +(-1)^2 = 1 + +---1 = -1 + +1 - 2 - 3 = -4 +1 - (2 - 3) = 2 + +2^2^3 = 256 +(2^2)^3 = 64 +]]) + +# Compose pieces to build the actual .y file. +AT_DATA([Calc.y],[[/* Infix notation calculator--calc */ +%language "Java" +%name-prefix "Calc" +%define parser_class_name {Calc} + +%code { +static class UserLexer implements Calc.Lexer +{ + StreamTokenizer st; + StringReader rdr; + + public UserLexer(StringReader reader) + { + rdr = reader; + st = new StreamTokenizer(rdr); + st.resetSyntax(); + st.eolIsSignificant(true); + st.whitespaceChars(9, 9); + st.whitespaceChars(32, 32); + st.wordChars(48, 57); + } + + Integer yylval; + + public Object getLVal() { return yylval; } + + public void yyerror(String msg) { System.err.println(msg); } + + public int yylex () throws IOException + { + switch (st.nextToken()) { + case StreamTokenizer.TT_EOF: return EOF; + case StreamTokenizer.TT_EOL: return (int) '\n'; + case StreamTokenizer.TT_WORD: + yylval = new Integer (st.sval); + return NUM; + default: return st.ttype; + } + } +} + +} + +%code { +public static void main (String[] argv) + throws IOException +{ + StringReader reader = getinput(argv[0]); + UserLexer lexer = new UserLexer(reader); + Calc calc = new Calc(lexer); + calc.setDebugLevel(1); + calc.parse(); +}//main + +} + +]AT_CALC_BODY[ + +]]) + +# This data was captured from running a pull parser. +AT_DATA([[expout]],[[Stack now 0 +Stack now 0 2 +Stack now 0 9 +Stack now 0 9 19 +Stack now 0 9 19 2 +Stack now 0 9 19 28 +Stack now 0 9 19 28 20 +Stack now 0 9 19 28 20 2 +Stack now 0 9 19 28 20 29 +Stack now 0 9 19 28 +Stack now 0 9 +Stack now 0 9 17 +Stack now 0 9 17 2 +Stack now 0 9 17 26 +Stack now 0 9 +Stack now 0 9 23 +Stack now 0 8 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 19 +Stack now 0 7 9 19 2 +Stack now 0 7 9 19 28 +Stack now 0 7 9 19 28 20 +Stack now 0 7 9 19 28 20 3 +Stack now 0 7 9 19 28 20 3 2 +Stack now 0 7 9 19 28 20 3 12 +Stack now 0 7 9 19 28 20 29 +Stack now 0 7 9 19 28 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 3 +Stack now 0 7 3 2 +Stack now 0 7 3 12 +Stack now 0 7 3 12 22 +Stack now 0 7 3 12 22 2 +Stack now 0 7 3 12 22 31 +Stack now 0 7 3 12 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 5 +Stack now 0 7 5 3 +Stack now 0 7 5 3 2 +Stack now 0 7 5 3 12 +Stack now 0 7 5 14 +Stack now 0 7 5 14 25 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 3 +Stack now 0 7 3 3 +Stack now 0 7 3 3 3 +Stack now 0 7 3 3 3 2 +Stack now 0 7 3 3 3 12 +Stack now 0 7 3 3 12 +Stack now 0 7 3 12 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 2 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 2 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 5 +Stack now 0 7 9 18 5 2 +Stack now 0 7 9 18 5 14 +Stack now 0 7 9 18 5 14 18 +Stack now 0 7 9 18 5 14 18 2 +Stack now 0 7 9 18 5 14 18 27 +Stack now 0 7 9 18 5 14 +Stack now 0 7 9 18 5 14 25 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 22 31 22 +Stack now 0 7 9 22 31 22 2 +Stack now 0 7 9 22 31 22 31 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 5 +Stack now 0 7 5 2 +Stack now 0 7 5 14 +Stack now 0 7 5 14 22 +Stack now 0 7 5 14 22 2 +Stack now 0 7 5 14 22 31 +Stack now 0 7 5 14 +Stack now 0 7 5 14 25 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 15 +]]) + +AT_BISON_CHECK([PUSHPULLFLAG [-o Calc.java Calc.y]]) +AT_JAVA_COMPILE([[Calc.java]]) +#Verify that this is a push parser. +AT_CHECK_JAVA_GREP([[Calc.java]], + [[.*public void push_parse_initialize().*]]) +# Capture stderr output for comparison purposes. +AT_JAVA_PARSER_CHECK([Calc input], 0, [ignore-nolog], [stderr-nolog]) +# Extract the "Stack Now" lines from the error output, +# send them to stdout (via the sed command) and compare to expout. +# NOTE: because the target is "expout", this macro automatically +# compares the output of the sed command with the contents of +# the file "expout" (defined above). +AT_CHECK([[sed -e '/^Stack now.*$/p' -e d ./stderr|tee jj]],[ignore],[expout],[ignore-nolog]) +AT_BISON_OPTION_POPDEFS +AT_CLEANUP + +# This test looks for location reporting by looking +# at the lexer output with locations enabled. +# It defines a lexer that reports location info. +AT_SETUP([Calc parser with %locations %code lexer and api.push-pull both]) +AT_BISON_OPTION_PUSHDEFS + +AT_DATA([Calc.y],[[/* Infix notation calculator--calc. */ +%language "Java" +%name-prefix "Calc" +%define parser_class_name {Calc} +%lex-param { Reader rdr } +%locations + +%code imports { + import java.io.*; +} + +%code lexer { + StreamTokenizer st; + Integer yylval; + + public YYLexer(Reader rdr) + { + st = new StreamTokenizer(rdr); + st.resetSyntax(); + st.eolIsSignificant(true); + st.whitespaceChars(9, 9); + st.whitespaceChars(32, 32); + st.wordChars(48, 57); + } + + Position yypos = new Position (1, 0); + + public Position getStartPos() { return yypos; } + + public Position getEndPos() { return yypos; } + + public Object getLVal() { return yylval; } + + public void yyerror(Location loc, String msg) + { + System.err.println(loc+":"+msg); + } + + public int yylex () throws IOException + { + yypos = new Position (yypos.lineno (),yypos.token () + 1); + switch (st.nextToken()) { + case StreamTokenizer.TT_EOF: + return EOF; + case StreamTokenizer.TT_EOL: + yypos = new Position (yypos.lineno () + 1, 0); + return (int) '\n'; + case StreamTokenizer.TT_WORD: + yylval = new Integer (st.sval); + return NUM; + default: + return st.ttype; + } + } +} + +%code { +class Position { + public int line; + public int token; + + public Position () { line = 0; token = 0; } + + public Position (int l, int t) { line = l; token = t; } + + public boolean equals (Position l) + { + return l.line == line && l.token == token; + } + + public String toString () + { + return Integer.toString(line) + "." + Integer.toString(token); + } + + public int lineno () { return line; } + + public int token () { return token; } +}//Class Position +} + +%code { +public static void main (String[] argv) + throws IOException +{ + StringReader reader = getinput(argv[0]); + Calc calc = new Calc(reader); + calc.setDebugLevel(1); + calc.parse(); +} +} + +]AT_CALC_BODY[ + +]]) + +# Define the expected calculator output. +# This should match the output from a pull parser. +AT_DATA([output],[[total = 7 +total = -5 +total = -1 +total = 1 +total = -1 +total = -4 +total = 2 +total = 256 +total = 64 +]]) + +AT_DATA([locations],[[Next token is token "number" (1.1: 1) +Next token is token '+' (1.2: 1) +Next token is token "number" (1.3: 2) +Next token is token '*' (1.4: 2) +Next token is token "number" (1.5: 3) +Next token is token '=' (1.6: 3) +Next token is token '=' (1.6: 3) +Next token is token '=' (1.6: 3) +Next token is token "number" (1.7: 7) +Next token is token '\n' (2.0: 7) +Next token is token '\n' (2.0: 7) +Next token is token "number" (2.1: 1) +Next token is token '+' (2.2: 1) +Next token is token "number" (2.3: 2) +Next token is token '*' (2.4: 2) +Next token is token '-' (2.5: 2) +Next token is token "number" (2.6: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '-' (2.8: 3) +Next token is token "number" (2.9: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (4.0: 5) +Next token is token '-' (4.1: 5) +Next token is token "number" (4.2: 1) +Next token is token '^' (4.3: 1) +Next token is token "number" (4.4: 2) +Next token is token '=' (4.5: 2) +Next token is token '=' (4.5: 2) +Next token is token '=' (4.5: 2) +Next token is token '-' (4.6: 2) +Next token is token "number" (4.7: 1) +Next token is token '\n' (5.0: 1) +Next token is token '\n' (5.0: 1) +Next token is token '\n' (5.0: 1) +Next token is token '(' (5.1: 1) +Next token is token '-' (5.2: 1) +Next token is token "number" (5.3: 1) +Next token is token ')' (5.4: 1) +Next token is token ')' (5.4: 1) +Next token is token '^' (5.5: 1) +Next token is token "number" (5.6: 2) +Next token is token '=' (5.7: 2) +Next token is token '=' (5.7: 2) +Next token is token "number" (5.8: 1) +Next token is token '\n' (6.0: 1) +Next token is token '\n' (6.0: 1) +Next token is token '\n' (7.0: 1) +Next token is token '-' (7.1: 1) +Next token is token '-' (7.2: 1) +Next token is token '-' (7.3: 1) +Next token is token "number" (7.4: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '-' (7.6: 1) +Next token is token "number" (7.7: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (9.0: 1) +Next token is token "number" (9.1: 1) +Next token is token '-' (9.2: 1) +Next token is token "number" (9.3: 2) +Next token is token '-' (9.4: 2) +Next token is token '-' (9.4: 2) +Next token is token "number" (9.5: 3) +Next token is token '=' (9.6: 3) +Next token is token '=' (9.6: 3) +Next token is token '-' (9.7: 3) +Next token is token "number" (9.8: 4) +Next token is token '\n' (10.0: 4) +Next token is token '\n' (10.0: 4) +Next token is token '\n' (10.0: 4) +Next token is token "number" (10.1: 1) +Next token is token '-' (10.2: 1) +Next token is token '(' (10.3: 1) +Next token is token "number" (10.4: 2) +Next token is token '-' (10.5: 2) +Next token is token "number" (10.6: 3) +Next token is token ')' (10.7: 3) +Next token is token ')' (10.7: 3) +Next token is token '=' (10.8: 3) +Next token is token '=' (10.8: 3) +Next token is token "number" (10.9: 2) +Next token is token '\n' (11.0: 2) +Next token is token '\n' (11.0: 2) +Next token is token '\n' (12.0: 2) +Next token is token "number" (12.1: 2) +Next token is token '^' (12.2: 2) +Next token is token "number" (12.3: 2) +Next token is token '^' (12.4: 2) +Next token is token "number" (12.5: 3) +Next token is token '=' (12.6: 3) +Next token is token '=' (12.6: 3) +Next token is token '=' (12.6: 3) +Next token is token "number" (12.7: 256) +Next token is token '\n' (13.0: 256) +Next token is token '\n' (13.0: 256) +Next token is token '(' (13.1: 256) +Next token is token "number" (13.2: 2) +Next token is token '^' (13.3: 2) +Next token is token "number" (13.4: 2) +Next token is token ')' (13.5: 2) +Next token is token ')' (13.5: 2) +Next token is token '^' (13.6: 2) +Next token is token "number" (13.7: 3) +Next token is token '=' (13.8: 3) +Next token is token '=' (13.8: 3) +Next token is token "number" (13.9: 64) +Next token is token '\n' (14.0: 64) +Next token is token '\n' (14.0: 64) +]]) + +# Define the calculator input. +# Warning: if you changes the input file +# then the locations test file position numbers +# may be incorrect and you will have +# to modify that file as well. + +AT_DATA([input],[[1 + 2 * 3 = 7 +1 + 2 * -3 = -5 + +-1^2 = -1 +(-1)^2 = 1 + +---1 = -1 + +1 - 2 - 3 = -4 +1 - (2 - 3) = 2 + +2^2^3 = 256 +(2^2)^3 = 64 +]]) + +AT_BISON_CHECK([PUSHPULLFLAG [-o Calc.java Calc.y]]) +AT_JAVA_COMPILE([[Calc.java]]) +# Verify that this is a push parser +AT_CHECK_JAVA_GREP([[Calc.java]], + [[.*public void push_parse_initialize().*]]) +# Capture the stdout and stderr output for comparison purposes. +AT_JAVA_PARSER_CHECK([Calc input], 0, [stdout-nolog], [stderr-nolog]) +# 1. Check that the token locations are correct +AT_CHECK([[cp -f ./locations ./expout]],[ignore],[ignore-nolog],[ignore-nolog]) +AT_CHECK([[sed -e '/^Next token.*$/p' -e d ./stderr]],[ignore],[expout],[ignore-nolog]) +# 2. Check that the calculator output matches that of a pull parser +AT_CHECK([[rm -f ./expout; cp -f ./output ./expout]],[ignore],[ignore-nolog],[ignore-nolog]) +AT_CHECK([[cat ./stdout]],[ignore],[expout],[ignore-nolog]) +AT_CLEANUP diff --git a/tests/local.mk b/tests/local.mk index 5299c2e4..5f7fa452 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -53,6 +53,7 @@ TESTSUITE_AT = \ tests/headers.at \ tests/input.at \ tests/java.at \ + tests/javapush.at \ tests/local.at \ tests/named-refs.at \ tests/output.at \ diff --git a/tests/testsuite.at b/tests/testsuite.at index 4c995132..47913add 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -63,17 +63,18 @@ m4_include([existing.at]) # Some old bugs. m4_include([regression.at]) +# Push parsing specific tests. +m4_include([push.at]) + # Some C++ specific tests. m4_include([c++.at]) # And some Java specific tests. m4_include([java.at]) +m4_include([javapush.at]) # GLR tests: # C++ types, simplified m4_include([cxx-type.at]) # Regression tests m4_include([glr-regression.at]) - -# Push parsing specific tests. -m4_include([push.at]) |