Updated parser to tokenize and parse line by line.

Lexer tokenizes the current line whenever it is called by parser.
"end" token is returned when EOF is encountered.
Currently assumes that newline is their before EOF.

6 files changed, 46 insertions, 48 deletions

diff --git a/lib/Automake/Parser/Lexer.pm b/lib/Automake/Parser/Lexer.pm
index 1b8a22777..ec514c50f 100644
--- a/lib/Automake/Parser/Lexer.pm
+++ b/lib/Automake/Parser/Lexer.pm
@@ -5,18 +5,25 @@ use Exporter;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(lex);
 
+# Store end token.
+my @end_tok = ["end"];
+
 # lex(string,multiline)
 # Takes as input a string of line and multiline variable deciding whether 
 # current line is related to the previous line. Divides it into tokens as 
 # specified by Regex and outputs an array of Tokens. Every Token is an 
 # array having two values: token name and its value. If its an operator, 
 # it has only one value.
-sub lex($$)
+sub lex($)
 {
-	my ( $curr_line , $multiline ) = @_;
+	my ( $multiline ) = @_;
 	my @tokens;
 	my $rhs = 0;
-	$_ = $curr_line;
+    $_ = scalar <>;
+	
+	# Send an end token when EOF is reached.
+	return ( \@end_tok, $multiline ) unless $_;
+	
 	while( $_ )
 	{
 		if( $multiline )
@@ -130,6 +137,10 @@ sub lex($$)
 			die "Incorrect input $_";
 		}
 	}
+	
+	# Returns undef when no tokens.
+	return ( undef, $multiline ) if !@tokens;
+	
 	return ( \@tokens , $multiline );
 }
 
diff --git a/lib/Automake/Parser/ParserTable.pm b/lib/Automake/Parser/ParserTable.pm
index 13b607821..1647af076 100644
--- a/lib/Automake/Parser/ParserTable.pm
+++ b/lib/Automake/Parser/ParserTable.pm
@@ -9,21 +9,21 @@ our @Export=qw(@table $accept);
 our $accept=17;
 
 our @table=(
-		{stmts => 5, makerule => 8, comment => 2, lhs => 11, input => 4, ifblock => 10, stmt => 6, optionlist => 13, automakerule => 7, conditional => 9, if => 3, commentlist => 12, value => 1},
-		{reduce => [1,  \&lhs], ':' => 14, '_' => 15},
+		{makerule => 8, stmt => 6, value => 1, input => 4, if => 3, automakerule => 7, ifblock => 10, comment => 2, optionlist => 13, lhs => 11, stmts => 5, conditional => 9, commentlist => 12},
+		{'_' => 15, reduce => [1,  \&lhs], ':' => 14},
 		{reduce => [1,  \&commentlist]},
 		{value => 16},
 		{end => 17},
-		{makerule => 8, comment => 2, lhs => 11, ifblock => 10, stmt => 18, optionlist => 13, automakerule => 7, conditional => 9, if => 3, reduce => [1,  \&input], commentlist => 12, value => 1},
+		{optionlist => 13, comment => 2, commentlist => 12, lhs => 11, conditional => 9, automakerule => 7, ifblock => 10, reduce => [1,  \&input], if => 3, stmt => 18, makerule => 8, value => 1},
 		{newline => 19},
 		{reduce => [1,  \&stmt]},
 		{reduce => [1,  \&stmt]},
 		{reduce => [1,  \&stmt]},
-		{optionalelse => 21, reduce => [0,  \&optionalelse], else => 20},
+		{else => 20, reduce => [0,  \&optionalelse], optionalelse => 21},
 		{'+' => 23, '=' => 22},
 		{comment => 24, reduce => [1,  \&stmt]},
-		{PYTHON => 30, SCRIPTS => 32, JAVA => 31, LTLIBRARIES => 28, DATA => 33, value => 25, HEADERS => 34, TEXINFOS => 36, primaries => 37, LIBRARIES => 27, PROGRAMS => 26, LISP => 29, MASN => 35},
-		{rhs => 39, rhsval => 38},
+		{primaries => 37, value => 25, PROGRAMS => 26, MASN => 35, TEXINFOS => 36, LIBRARIES => 27, HEADERS => 34, SCRIPTS => 32, DATA => 33, LTLIBRARIES => 28, LISP => 29, JAVA => 31, PYTHON => 30},
+		{rhsval => 38, rhs => 39},
 		{reduce => [2,  \&optionlist]},
 		{newline => 40},
 		{},
@@ -49,21 +49,21 @@ our @table=(
 		{reduce => [2,  \&lhs]},
 		{reduce => [1,  \&rhs]},
 		{reduce => [3,  \&makerule], rhsval => 48},
-		{lhs => 11, ifblock => 10, stmts => 49, makerule => 8, comment => 2, conditional => 9, if => 3, commentlist => 12, value => 1, stmt => 6, optionlist => 13, automakerule => 7},
+		{automakerule => 7, ifblock => 10, comment => 2, optionlist => 13, commentlist => 12, lhs => 11, stmts => 49, conditional => 9, stmt => 6, makerule => 8, value => 1, if => 3},
 		{reduce => [3,  \&stmts]},
-		{comment => 2, stmts => 50, makerule => 8, ifblock => 10, lhs => 11, automakerule => 7, optionlist => 13, stmt => 6, value => 1, commentlist => 12, if => 3, conditional => 9},
-		{reduce => [0,  \&optionalcond], value => 51, optionalcond => 52},
-		{optionalcomments => 53, comment => 2, commentlist => 54, reduce => [0,  \&optionalcomments]},
-		{reduce => [1,  \&optionalrhs], rhsval => 48},
-		{rhs => 45, optionalrhs => 55, reduce => [0,  \&optionalrhs], rhsval => 38},
+		{if => 3, value => 1, stmt => 6, makerule => 8, commentlist => 12, lhs => 11, stmts => 50, conditional => 9, comment => 2, optionlist => 13, ifblock => 10, automakerule => 7},
+		{value => 51, optionalcond => 52, reduce => [0,  \&optionalcond]},
+		{comment => 2, reduce => [0,  \&optionalcomments], optionalcomments => 53, commentlist => 54},
+		{rhsval => 48, reduce => [1,  \&optionalrhs]},
+		{optionalrhs => 55, rhsval => 38, reduce => [0,  \&optionalrhs], rhs => 45},
 		{reduce => [3,  \&optionlist]},
 		{reduce => [2,  \&rhs]},
-		{reduce => [4,  \&ifblock], value => 1, commentlist => 12, if => 3, conditional => 9, automakerule => 7, optionlist => 13, stmt => 18, ifblock => 10, lhs => 11, comment => 2, makerule => 8},
-		{comment => 2, makerule => 8, ifblock => 10, lhs => 11, automakerule => 7, optionlist => 13, stmt => 18, reduce => [3,  \&optionalelse], value => 1, commentlist => 12, if => 3, conditional => 9},
+		{if => 3, reduce => [4,  \&ifblock], value => 1, stmt => 18, makerule => 8, commentlist => 12, conditional => 9, lhs => 11, optionlist => 13, comment => 2, ifblock => 10, automakerule => 7},
+		{value => 1, stmt => 18, makerule => 8, if => 3, reduce => [3,  \&optionalelse], ifblock => 10, automakerule => 7, commentlist => 12, conditional => 9, lhs => 11, optionlist => 13, comment => 2},
 		{reduce => [1,  \&optionalcond]},
 		{reduce => [4,  \&conditional]},
 		{reduce => [4,  \&automakerule]},
-		{reduce => [1,  \&optionalcomments], comment => 24},
-		{reduce => [0,  \&optionalcomments], commentlist => 54, comment => 2, optionalcomments => 56},
+		{comment => 24, reduce => [1,  \&optionalcomments]},
+		{comment => 2, reduce => [0,  \&optionalcomments], optionalcomments => 56, commentlist => 54},
 		{reduce => [5,  \&automakerule]}
 );
\ No newline at end of file
diff --git a/lib/Automake/Parser/input.txt b/lib/Automake/Parser/input.txt
index 77e2ba539..9324ebbca 100644
--- a/lib/Automake/Parser/input.txt
+++ b/lib/Automake/Parser/input.txt
@@ -1,4 +1,4 @@
-## Process this file with automake to produce Makefile.in
+# Process this file with automake to produce Makefile.in
 dist_bin_PROGRAMS = server \
 client
 server_SOURCES = server.c db.c   ## Server Files \
diff --git a/lib/Automake/Parser/parser.pl b/lib/Automake/Parser/parser.pl
index 65eda46da..993445004 100644
--- a/lib/Automake/Parser/parser.pl
+++ b/lib/Automake/Parser/parser.pl
@@ -10,35 +10,12 @@ my $debug = 0;
 #Stores the list of tokens generated by lexer.
 my @tokens; 
 
-my $multiline = 0;
-my $curr_tokens;
-
-#Read input from file specified in Arguements or STDIN.
-while ( <> )
-{
-	( $curr_tokens, $multiline ) = lex( $_ , $multiline );
-	push @tokens, @$curr_tokens;
-}
-
-#Prints to STDERR if Debug mode is on.
-if( $debug )
-{
-	print STDERR "Lexer Output\n";
-	foreach my $token ( @tokens )
-	{
-		print STDERR join(" ", @{$token}), "\n";
-	}
-}
-#Push a newline token if last token is not newline
-if( $tokens[-1][0] ne "newline" )
-{
-	push @tokens,["newline"];
-}
-
-push @tokens, [ "end" ];
 my @stack = (0);
 print STDERR "Parser Output\n" if $debug;
 
+my $multiline = 0;
+my $curr_tokens;
+
 while ( @stack )
 {
 	if($stack[-1] == $ParserTable::accept)
@@ -47,6 +24,16 @@ while ( @stack )
 		printgraph( $stack[-4] );
 		last;
 	}
+	while( !@tokens )
+	{
+		# Calls lexer to get next tokens.
+		( $curr_tokens, $multiline ) = lex( $multiline );
+		
+		# Continue if their is no tokens
+		next unless $curr_tokens;
+		
+		push @tokens, @$curr_tokens;
+	}
 	my @curr_token = @{ $tokens[0] };	
 	if(my $val = $ParserTable::table[ $stack[-1] ]{ $curr_token[0] })
 	{
diff --git a/lib/Automake/Parser/t/conditional.txt b/lib/Automake/Parser/t/conditional.txt
index 3708d8093..1db874a37 100644
--- a/lib/Automake/Parser/t/conditional.txt
+++ b/lib/Automake/Parser/t/conditional.txt
@@ -3,4 +3,4 @@ if windows
 server_SOURCES = winserver.c
 else
 server_SOURCES = unixserver.c
-endif
\ No newline at end of file
+endif
diff --git a/lib/Automake/Parser/t/nestedcond.txt b/lib/Automake/Parser/t/nestedcond.txt
index 99d9ae9ca..b7141a5e6 100644
--- a/lib/Automake/Parser/t/nestedcond.txt
+++ b/lib/Automake/Parser/t/nestedcond.txt
@@ -17,4 +17,4 @@ endif
 if CASE_B
 SUBDIRS += iXYZ
 SUBDIRS += jZYX
-endif
\ No newline at end of file
+endif