merge perly.y and madlu.y

p4raw-id: //depot/perl@29455

1 files changed, 736 insertions, 220 deletions

diff --git a/perly.y b/perly.y
index 77caeb6892..f5b4203977 100644
--- a/perly.y
+++ b/perly.y
@@ -1,6 +1,6 @@
 /*    perly.y
  *
- *    Copyright (c) 1991-2002, 2003, 2004 Larry Wall
+ *    Copyright (c) 1991-2002, 2003, 2004, 2005, 2006 Larry Wall
  *
  *    You may distribute under the terms of either the GNU General Public
  *    License or the Artistic License, as specified in the README file.
@@ -10,12 +10,25 @@
 /*
  * 'I see,' laughed Strider.  'I look foul and feel fair.  Is that it?
  * All that is gold does not glitter, not all those who wander are lost.'
- */
-
-/* This file holds the grammar for the Perl language. If edited, you need
+ *
+ * This file holds the grammar for the Perl language. If edited, you need
  * to run regen_perly.pl, which re-creates the files perly.h, perly.tab
  * and perly.act which are derived from this.
  *
+ * Note that these derived files are included and compiled twice; once
+ * from perly.c, and once from madly.c. The second time, a number of MAD
+ * macros are defined, which compile in extra code that allows the parse
+ * tree to be accurately dumped. In particular:
+ *
+ * MAD            defined if compiling madly.c
+ * DO_MAD(A)      expands to A  under madly.c, to null otherwise
+ * IF_MAD(a,b)    expands to A under madly.c, to B otherwise
+ * TOKEN_GETMAD() expands to token_getmad() under madly.c, to null otherwise
+ * TOKEN_FREE()   similarly
+ * OP_GETMAD()    similarly
+ * IVAL(i)        expands to (i)->tk_lval.ival or (i)
+ * PVAL(p)        expands to (p)->tk_lval.pval or (p)
+ *
  * The main job of of this grammar is to call the various newFOO()
  * functions in op.c to build a syntax tree of OP structs.
  * It relies on the lexer in toke.c to do the tokenizing.
@@ -25,77 +38,88 @@
 
 %pure_parser
 
+/* FIXME for MAD - is the new mintro on while and until important?  */
+
 %start prog
 
 %union {
     I32	ival;
     char *pval;
-#ifdef PERL_MAD
-    TOKEN* tkval;
-#endif
     OP *opval;
     GV *gvval;
+#ifdef PERL_IN_MADLY_C
+    TOKEN* p_tkval;
+    TOKEN* tkval;
+#else
+    char *p_tkval;
+    I32	tkval;
+#endif
 }
 
-%token <ival> '{'
+%token <tkval> '{' '}' '[' ']' '-' '+' '$' '@' '%' '*' '&' ';'
 
 %token <opval> WORD METHOD FUNCMETH THING PMFUNC PRIVATEREF
 %token <opval> FUNC0SUB UNIOPSUB LSTOPSUB
-%token <pval> LABEL
-%token <ival> FORMAT SUB ANONSUB PACKAGE USE
-%token <ival> WHILE UNTIL IF UNLESS ELSE ELSIF CONTINUE FOR
-%token <ival> GIVEN WHEN DEFAULT
-%token <ival> LOOPEX DOTDOT
-%token <ival> FUNC0 FUNC1 FUNC UNIOP LSTOP
-%token <ival> RELOP EQOP MULOP ADDOP
-%token <ival> DOLSHARP DO HASHBRACK NOAMP
-%token <ival> LOCAL MY MYSUB REQUIRE
-%token COLONATTR
-
-%type <ival> prog decl format startsub startanonsub startformsub mintro
-%type <ival> progstart remember mremember '&' savescope mydefsv
+%token <p_tkval> LABEL
+%token <tkval> FORMAT SUB ANONSUB PACKAGE USE
+%token <tkval> WHILE UNTIL IF UNLESS ELSE ELSIF CONTINUE FOR
+%token <tkval> GIVEN WHEN DEFAULT
+%token <tkval> LOOPEX DOTDOT
+%token <tkval> FUNC0 FUNC1 FUNC UNIOP LSTOP
+%token <tkval> RELOP EQOP MULOP ADDOP
+%token <tkval> DOLSHARP DO HASHBRACK NOAMP
+%token <tkval> LOCAL MY MYSUB REQUIRE
+%token <tkval> COLONATTR
+
+%type <ival> prog progstart remember mremember savescope
+%type <ival>  startsub startanonsub startformsub
+/* FIXME for MAD - are these two ival? */
+%type <ival> mydefsv mintro
+
+%type <opval> decl format subrout mysubrout package use peg
+
 %type <opval> block mblock lineseq line loop cond else
 %type <opval> expr term subscripted scalar ary hsh arylen star amper sideff
 %type <opval> argexpr nexpr texpr iexpr mexpr mnexpr miexpr
 %type <opval> listexpr listexprcom indirob listop method
 %type <opval> formname subname proto subbody cont my_scalar
-%type <opval> subattrlist myattrlist mysubrout myattrterm myterm
+%type <opval> subattrlist myattrlist myattrterm myterm
 %type <opval> termbinop termunop anonymous termdo
 %type <opval> switch case
-%type <pval> label
+%type <p_tkval> label
 
-%nonassoc PREC_LOW
+%nonassoc <tkval> PREC_LOW
 %nonassoc LOOPEX
 
-%left <ival> OROP DOROP
-%left ANDOP
-%right NOTOP
+%left <tkval> OROP DOROP
+%left <tkval> ANDOP
+%right <tkval> NOTOP
 %nonassoc LSTOP LSTOPSUB
-%left ','
-%right <ival> ASSIGNOP
-%right '?' ':'
+%left <tkval> ','
+%right <tkval> ASSIGNOP
+%right <tkval> '?' ':'
 %nonassoc DOTDOT
-%left OROR DORDOR
-%left ANDAND
-%left <ival> BITOROP
-%left <ival> BITANDOP
+%left <tkval> OROR DORDOR
+%left <tkval> ANDAND
+%left <tkval> BITOROP
+%left <tkval> BITANDOP
 %nonassoc EQOP
 %nonassoc RELOP
 %nonassoc UNIOP UNIOPSUB
 %nonassoc REQUIRE
-%left <ival> SHIFTOP
+%left <tkval> SHIFTOP
 %left ADDOP
 %left MULOP
-%left <ival> MATCHOP
-%right '!' '~' UMINUS REFGEN
-%right <ival> POWOP
-%nonassoc PREINC PREDEC POSTINC POSTDEC
-%left ARROW
-%nonassoc <ival> ')'
-%left '('
+%left <tkval> MATCHOP
+%right <tkval> '!' '~' UMINUS REFGEN
+%right <tkval> POWOP
+%nonassoc <tkval> PREINC PREDEC POSTINC POSTDEC
+%left <tkval> ARROW
+%nonassoc <tkval> ')'
+%left <tkval> '('
 %left '[' '{'
 
-%token PEG
+%token <tkval> PEG
 
 %% /* RULES */
 
@@ -107,9 +131,12 @@ prog	:	progstart
 
 /* An ordinary block */
 block	:	'{' remember lineseq '}'
-			{ if (PL_copline > (line_t)$1)
-			      PL_copline = (line_t)$1;
-			  $$ = block_end($2, $3); }
+			{ if (PL_copline > (line_t)IVAL($1))
+			      PL_copline = (line_t)IVAL($1);
+			  $$ = block_end($2, $3);
+			  TOKEN_GETMAD($1,$$,'{');
+			  TOKEN_GETMAD($4,$$,'}');
+			}
 	;
 
 remember:	/* NULL */	/* start a full lexical scope */
@@ -128,9 +155,12 @@ progstart:
 
 
 mblock	:	'{' mremember lineseq '}'
-			{ if (PL_copline > (line_t)$1)
-			      PL_copline = (line_t)$1;
-			  $$ = block_end($2, $3); }
+			{ if (PL_copline > (line_t)IVAL($1))
+			      PL_copline = (line_t)IVAL($1);
+			  $$ = block_end($2, $3);
+			  TOKEN_GETMAD($1,$$,'{');
+			  TOKEN_GETMAD($4,$$,'}');
+			}
 	;
 
 mremember:	/* NULL */	/* start a partial lexical scope */
@@ -144,7 +174,12 @@ savescope:	/* NULL */	/* remember stack pos in case of error */
 lineseq	:	/* NULL */
 			{ $$ = Nullop; }
 	|	lineseq decl
-			{ $$ = $1; }
+			{
+			$$ = IF_MAD(
+				append_list(OP_LINESEQ,
+			    	    (LISTOP*)$1, (LISTOP*)$2),
+				$1);
+			}
 	|	lineseq savescope line
 			{   LEAVE_SCOPE($2);
 			    $$ = append_list(OP_LINESEQ,
@@ -155,24 +190,46 @@ lineseq	:	/* NULL */
 
 /* A "line" in the program */
 line	:	label cond
-			{ $$ = newSTATEOP(0, $1, $2); }
+			{ $$ = newSTATEOP(0, PVAL($1), $2);
+			  TOKEN_GETMAD($1,((LISTOP*)$$)->op_first,'L'); }
 	|	loop	/* loops add their own labels */
 	|	switch  /* ... and so do switches */
 			{ $$ = $1; }
 	|	label case
-			{ $$ = newSTATEOP(0, $1, $2); }
+			{ $$ = newSTATEOP(0, PVAL($1), $2); }
 	|	label ';'
-			{ if ($1 != Nullch) {
-			      $$ = newSTATEOP(0, $1, newOP(OP_NULL, 0));
-			    }
-			    else {
-			      $$ = Nullop;
-			      PL_copline = NOLINE;
-			    }
-			    PL_expect = XSTATE; }
+			{
+			  if (PVAL($1)) {
+			      $$ = newSTATEOP(0, PVAL($1), newOP(OP_NULL, 0));
+			      TOKEN_GETMAD($1,$$,'L');
+			      TOKEN_GETMAD($2,((LISTOP*)$$)->op_first,';');
+			  }
+			  else {
+			      $$ = IF_MAD(
+					newOP(OP_NULL, 0),
+					Nullop);
+                              PL_copline = NOLINE;
+			      TOKEN_FREE($1);
+			      TOKEN_GETMAD($2,$$,';');
+			  }
+			  PL_expect = XSTATE;
+			}
 	|	label sideff ';'
-			{ $$ = newSTATEOP(0, $1, $2);
-			  PL_expect = XSTATE; }
+			{
+			  $$ = newSTATEOP(0, PVAL($1), $2);
+			  PL_expect = XSTATE;
+			  DO_MAD(
+			      /* sideff might already have a nexstate */
+			      OP* op = ((LISTOP*)$$)->op_first;
+			      if (op) {
+				  while (op->op_sibling &&
+				     op->op_sibling->op_type == OP_NEXTSTATE)
+					op = op->op_sibling;
+				  token_getmad($1,op,'L');
+				  token_getmad($3,op,';');
+			      }
+			  )
+			}
 	;
 
 /* An expression which may have a side-effect */
@@ -181,38 +238,62 @@ sideff	:	error
 	|	expr
 			{ $$ = $1; }
 	|	expr IF expr
-			{ $$ = newLOGOP(OP_AND, 0, $3, $1); }
+			{ $$ = newLOGOP(OP_AND, 0, $3, $1);
+			  TOKEN_GETMAD($2,$$,'i');
+			}
 	|	expr UNLESS expr
-			{ $$ = newLOGOP(OP_OR, 0, $3, $1); }
+			{ $$ = newLOGOP(OP_OR, 0, $3, $1);
+			  TOKEN_GETMAD($2,$$,'i');
+			}
 	|	expr WHILE expr
-			{ $$ = newLOOPOP(OPf_PARENS, 1, scalar($3), $1); }
+			{ $$ = newLOOPOP(OPf_PARENS, 1, scalar($3), $1);
+			  TOKEN_GETMAD($2,$$,'w');
+			}
 	|	expr UNTIL iexpr
-			{ $$ = newLOOPOP(OPf_PARENS, 1, $3, $1);}
+			{ $$ = newLOOPOP(OPf_PARENS, 1, $3, $1);
+			  TOKEN_GETMAD($2,$$,'w');
+			}
 	|	expr FOR expr
-			{ $$ = newFOROP(0, Nullch, (line_t)$2,
-					Nullop, $3, $1, Nullop); }
+			{ $$ = newFOROP(0, Nullch, (line_t)IVAL($2),
+					Nullop, $3, $1, Nullop);
+			  TOKEN_GETMAD($2,((LISTOP*)$$)->op_first->op_sibling,'w');
+			}
 	;
 
 /* else and elsif blocks */
 else	:	/* NULL */
 			{ $$ = Nullop; }
 	|	ELSE mblock
-			{ ($2)->op_flags |= OPf_PARENS; $$ = scope($2); }
+			{ ($2)->op_flags |= OPf_PARENS; $$ = scope($2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	ELSIF '(' mexpr ')' mblock else
-			{ PL_copline = (line_t)$1;
+			{ PL_copline = (line_t)IVAL($1);
 			    $$ = newCONDOP(0, $3, scope($5), $6);
-			    PL_hints |= HINT_BLOCK_SCOPE; }
+			    PL_hints |= HINT_BLOCK_SCOPE;
+			  TOKEN_GETMAD($1,$$,'I');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	;
 
 /* Real conditional expressions */
 cond	:	IF '(' remember mexpr ')' mblock else
-			{ PL_copline = (line_t)$1;
+			{ PL_copline = (line_t)IVAL($1);
 			    $$ = block_end($3,
-				   newCONDOP(0, $4, scope($6), $7)); }
+				   newCONDOP(0, $4, scope($6), $7));
+			  TOKEN_GETMAD($1,$$,'I');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 	|	UNLESS '(' remember miexpr ')' mblock else
-			{ PL_copline = (line_t)$1;
+			{ PL_copline = (line_t)IVAL($1);
 			    $$ = block_end($3,
-				   newCONDOP(0, $4, scope($6), $7)); }
+				   newCONDOP(0, $4, scope($6), $7));
+			  TOKEN_GETMAD($1,$$,'I');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 	;
 
 /* Cases for a switch statement */
@@ -227,41 +308,94 @@ case	:	WHEN '(' remember mexpr ')' mblock
 cont	:	/* NULL */
 			{ $$ = Nullop; }
 	|	CONTINUE block
-			{ $$ = scope($2); }
+			{ $$ = scope($2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	;
 
 /* Loops: while, until, for, and a bare block */
 loop	:	label WHILE '(' remember texpr ')' mintro mblock cont
-			{ PL_copline = (line_t)$2;
+			{ OP *innerop;
+			  PL_copline = (line_t)$2;
 			    $$ = block_end($4,
-				   newSTATEOP(0, $1,
-				     newWHILEOP(0, 1, (LOOP*)Nullop,
-						$2, $5, $8, $9, $7))); }
+				   newSTATEOP(0, PVAL($1),
+				     innerop = newWHILEOP(0, 1, (LOOP*)Nullop,
+						IVAL($2), $5, $8, $9, $7)));
+			  TOKEN_GETMAD($1,innerop,'L');
+			  TOKEN_GETMAD($2,innerop,'W');
+			  TOKEN_GETMAD($3,innerop,'(');
+			  TOKEN_GETMAD($6,innerop,')');
+			}
+
 	|	label UNTIL '(' remember iexpr ')' mintro mblock cont
-			{ PL_copline = (line_t)$2;
+			{ OP *innerop;
+			  PL_copline = (line_t)$2;
 			    $$ = block_end($4,
-				   newSTATEOP(0, $1,
-				     newWHILEOP(0, 1, (LOOP*)Nullop,
-						$2, $5, $8, $9, $7))); }
+				   newSTATEOP(0, PVAL($1),
+				     innerop = newWHILEOP(0, 1, (LOOP*)Nullop,
+						IVAL($2), $5, $8, $9, $7)));
+			  TOKEN_GETMAD($1,innerop,'L');
+			  TOKEN_GETMAD($2,innerop,'W');
+			  TOKEN_GETMAD($3,innerop,'(');
+			  TOKEN_GETMAD($6,innerop,')');
+			}
 	|	label FOR MY remember my_scalar '(' mexpr ')' mblock cont
-			{ $$ = block_end($4,
-				 newFOROP(0, $1, (line_t)$2, $5, $7, $9, $10)); }
+			{ OP *innerop;
+			  $$ = block_end($4,
+			     innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+					    $5, $7, $9, $10));
+			  TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+			  TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+			  TOKEN_GETMAD($3,((LISTOP*)innerop)->op_first->op_sibling,'d');
+			  TOKEN_GETMAD($6,((LISTOP*)innerop)->op_first->op_sibling,'(');
+			  TOKEN_GETMAD($8,((LISTOP*)innerop)->op_first->op_sibling,')');
+			}
 	|	label FOR scalar '(' remember mexpr ')' mblock cont
-			{ $$ = block_end($5,
-				 newFOROP(0, $1, (line_t)$2, mod($3, OP_ENTERLOOP),
-					  $6, $8, $9)); }
+			{ OP *innerop;
+			  $$ = block_end($5,
+			     innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+				    mod($3, OP_ENTERLOOP), $6, $8, $9));
+			  TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+			  TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+			  TOKEN_GETMAD($4,((LISTOP*)innerop)->op_first->op_sibling,'(');
+			  TOKEN_GETMAD($7,((LISTOP*)innerop)->op_first->op_sibling,')');
+			}
 	|	label FOR '(' remember mexpr ')' mblock cont
-			{ $$ = block_end($4,
-				 newFOROP(0, $1, (line_t)$2, Nullop, $5, $7, $8)); }
+			{ OP *innerop;
+			  $$ = block_end($4,
+			     innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+						    Nullop, $5, $7, $8));
+			  TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+			  TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+			  TOKEN_GETMAD($3,((LISTOP*)innerop)->op_first->op_sibling,'(');
+			  TOKEN_GETMAD($6,((LISTOP*)innerop)->op_first->op_sibling,')');
+			}
 	|	label FOR '(' remember mnexpr ';' texpr ';' mintro mnexpr ')'
 	    	    mblock
 			/* basically fake up an initialize-while lineseq */
 			{ OP *forop;
-			  PL_copline = (line_t)$2;
-			  forop = newSTATEOP(0, $1,
+			  PL_copline = (line_t)IVAL($2);
+			  forop = newSTATEOP(0, PVAL($1),
 					    newWHILEOP(0, 1, (LOOP*)Nullop,
-						$2, scalar($7),
+						IVAL($2), scalar($7),
 						$12, $10, $9));
+#ifdef MAD
+			  if (!$5)
+				$5 = newOP(OP_NULL, 0);
+			  forop = newUNOP(OP_NULL, 0, append_elem(OP_LINESEQ,
+				newSTATEOP(0,
+					   (($1)->tk_lval.pval
+					   ?savepv(($1)->tk_lval.pval):Nullch),
+					   $5),
+				forop));
+
+			  token_getmad($2,forop,'3');
+			  token_getmad($3,forop,'(');
+			  token_getmad($6,forop,'1');
+			  token_getmad($8,forop,'2');
+			  token_getmad($11,forop,')');
+			  token_getmad($1,forop,'L');
+#else
 			  if ($5) {
 				forop = append_elem(OP_LINESEQ,
                                         newSTATEOP(0, ($1?savepv($1):Nullch),
@@ -269,18 +403,21 @@ loop	:	label WHILE '(' remember texpr ')' mintro mblock cont
 					forop);
 			  }
 
+
+#endif
 			  $$ = block_end($4, forop); }
 	|	label block cont  /* a block is a loop that happens once */
-			{ $$ = newSTATEOP(0, $1,
+			{ $$ = newSTATEOP(0, PVAL($1),
 				 newWHILEOP(0, 1, (LOOP*)Nullop,
-					    NOLINE, Nullop, $2, $3, 0)); }
+					    NOLINE, Nullop, $2, $3, 0));
+			  TOKEN_GETMAD($1,((LISTOP*)$$)->op_first,'L'); }
 	;
 
 /* Switch blocks */
 switch	:	label GIVEN '(' remember mydefsv mexpr ')' mblock
 			{ PL_copline = (line_t) $2;
 			    $$ = block_end($4,
-				newSTATEOP(0, $1,
+				newSTATEOP(0, PVAL($1),
 				    newGIVENOP($6, scope($8),
 					(PADOFFSET) $5) )); }
 	;
@@ -299,7 +436,9 @@ nexpr	:	/* NULL */
 
 /* Boolean expression */
 texpr	:	/* NULL means true */
-			{ (void)scan_num("1", &yylval); $$ = yylval.opval; }
+			{ YYSTYPE tmplval;
+			  (void)scan_num("1", &tmplval);
+			  $$ = tmplval.opval; }
 	|	expr
 	;
 
@@ -323,26 +462,53 @@ miexpr	:	iexpr
 
 /* Optional "MAIN:"-style loop labels */
 label	:	/* empty */
-			{ $$ = Nullch; }
+			{
+#ifdef MAD
+			  YYSTYPE tmplval;
+			  tmplval.pval = Nullch;
+			  $$ = newTOKEN(OP_NULL, tmplval, 0);
+#else
+			  $$ = Nullch;
+#endif
+			}
 	|	LABEL
 	;
 
-/* Some kind of declaration - does not take part in the parse tree */
+/* Some kind of declaration - just hang on peg in the parse tree */
 decl	:	format
-			{ $$ = 0; }
+			{ $$ = $1; }
 	|	subrout
-			{ $$ = 0; }
+			{ $$ = $1; }
 	|	mysubrout
-			{ $$ = 0; }
+			{ $$ = $1; }
 	|	package
-			{ $$ = 0; }
+			{ $$ = $1; }
 	|	use
-			{ $$ = 0; }
+			{ $$ = $1; }
+
+    /* these two are only used by MAD */
+
+	|	peg
+			{ $$ = $1; }
+	;
+
+peg	:	PEG
+			{ $$ = newOP(OP_NULL,0);
+			  TOKEN_GETMAD($1,$$,'p');
+			}
 	;
 
 format	:	FORMAT startformsub formname block
 			{ SvREFCNT_inc(PL_compcv);
-			  newFORM($2, $3, $4); }
+#ifdef MAD
+			  $$ = newFORM($2, $3, $4);
+			  prepend_madprops($1->tk_mad, $$, 'F');
+			  $1->tk_mad = 0;
+			  token_free($1);
+#else
+			  newFORM($2, $3, $4);
+#endif
+			}
 	;
 
 formname:	WORD		{ $$ = $1; }
@@ -352,18 +518,40 @@ formname:	WORD		{ $$ = $1; }
 /* Unimplemented "my sub foo { }" */
 mysubrout:	MYSUB startsub subname proto subattrlist subbody
 			{ SvREFCNT_inc(PL_compcv);
-			  newMYSUB($2, $3, $4, $5, $6); }
+#ifdef MAD
+			  $$ = newMYSUB($2, $3, $4, $5, $6);
+			  token_getmad($1,$$,'d');
+#else
+			  newMYSUB($2, $3, $4, $5, $6);
+#endif
+			}
 	;
 
 /* Subroutine definition */
 subrout	:	SUB startsub subname proto subattrlist subbody
 			{ SvREFCNT_inc(PL_compcv);
-			  newATTRSUB($2, $3, $4, $5, $6); }
+#ifdef MAD
+			  OP* o = newSVOP(OP_ANONCODE, 0,
+			    (SV*)newATTRSUB($2, $3, $4, $5, $6));
+			  $$ = newOP(OP_NULL,0);
+			  op_getmad(o,$$,'&');
+			  op_getmad($3,$$,'n');
+			  op_getmad($4,$$,'s');
+			  op_getmad($5,$$,'a');
+			  token_getmad($1,$$,'d');
+			  append_madprops($6->op_madprop, $$, 0);
+			  $6->op_madprop = 0;
+#else
+			  newATTRSUB($2, $3, $4, $5, $6);
+			  $$ = Nullop;
+#endif
+			}
 	;
 
 startsub:	/* NULL */	/* start a regular subroutine scope */
 			{ $$ = start_subparse(FALSE, 0);
 			    SAVEFREESV(PL_compcv); }
+
 	;
 
 startanonsub:	/* NULL */	/* start an anonymous subroutine scope */
@@ -395,89 +583,169 @@ proto	:	/* NULL */
 subattrlist:	/* NULL */
 			{ $$ = Nullop; }
 	|	COLONATTR THING
-			{ $$ = $2; }
+			{ $$ = $2;
+			  TOKEN_GETMAD($1,$$,':');
+			}
 	|	COLONATTR
-			{ $$ = Nullop; }
+			{ $$ = IF_MAD(
+				    newOP(OP_NULL, 0),
+				    Nullop
+				);
+			  TOKEN_GETMAD($1,$$,':');
+			}
 	;
 
 /* List of attributes for a "my" variable declaration */
 myattrlist:	COLONATTR THING
-			{ $$ = $2; }
+			{ $$ = $2;
+			  TOKEN_GETMAD($1,$$,':');
+			}
 	|	COLONATTR
-			{ $$ = Nullop; }
+			{ $$ = IF_MAD(
+				    newOP(OP_NULL, 0),
+				    Nullop
+				);
+			  TOKEN_GETMAD($1,$$,':');
+			}
 	;
 
 /* Subroutine body - either null or a block */
 subbody	:	block	{ $$ = $1; }
-	|	';'	{ $$ = Nullop; PL_expect = XSTATE; }
+	|	';'	{ $$ = IF_MAD(
+				    newOP(OP_NULL,0),
+				    Nullop
+				);
+			  PL_expect = XSTATE;
+			  TOKEN_GETMAD($1,$$,';');
+			}
 	;
 
 package :	PACKAGE WORD ';'
-			{ package($2); }
+			{
+#ifdef MAD
+			  $$ = package($2);
+			  token_getmad($1,$$,'o');
+			  token_getmad($3,$$,';');
+#else
+			  package($2);
+#endif
+			}
 	;
 
 use	:	USE startsub
 			{ CvSPECIAL_on(PL_compcv); /* It's a BEGIN {} */ }
 		    WORD WORD listexpr ';'
-			 { SvREFCNT_inc(PL_compcv);
-			   utilize($1, $2, $4, $5, $6); }
+			{ SvREFCNT_inc(PL_compcv);
+#ifdef MAD
+			  $$ = utilize(IVAL($1), $2, $4, $5, $6);
+			  token_getmad($1,$$,'o');
+			  token_getmad($7,$$,';');
+			  if (PL_rsfp_filters && AvFILLp(PL_rsfp_filters) >= 0)
+			      append_madprops(newMADPROP('!', MAD_PV, "", 0), $$, 0);
+#else
+			  utilize(IVAL($1), $2, $4, $5, $6);
+#endif
+			}
 	;
 
 /* Ordinary expressions; logical combinations */
 expr	:	expr ANDOP expr
-			{ $$ = newLOGOP(OP_AND, 0, $1, $3); }
+			{ $$ = newLOGOP(OP_AND, 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	expr OROP expr
-			{ $$ = newLOGOP($2, 0, $1, $3); }
+			{ $$ = newLOGOP(IVAL($2), 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	expr DOROP expr
-			{ $$ = newLOGOP(OP_DOR, 0, $1, $3); }
+			{ $$ = newLOGOP(OP_DOR, 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	argexpr %prec PREC_LOW
 	;
 
 /* Expressions are a list of terms joined by commas */
 argexpr	:	argexpr ','
-			{ $$ = $1; }
+			{
+#ifdef MAD
+			  OP* op = newNULLLIST();
+			  token_getmad($2,op,',');
+			  $$ = append_elem(OP_LIST, $1, op);
+#else
+			  $$ = $1;
+#endif
+			}
 	|	argexpr ',' term
-			{ $$ = append_elem(OP_LIST, $1, $3); }
+			{ 
+			  DO_MAD(
+			      $3 = newUNOP(OP_NULL, 0, $3);
+			      token_getmad($2,$3,',');
+			  )
+			  $$ = append_elem(OP_LIST, $1, $3);
+			}
 	|	term %prec PREC_LOW
 	;
 
 /* List operators */
 listop	:	LSTOP indirob argexpr /* map {...} @args or print $fh @args */
-			{ $$ = convert($1, OPf_STACKED,
-				prepend_elem(OP_LIST, newGVREF($1,$2), $3) ); }
+			{ $$ = convert(IVAL($1), OPf_STACKED,
+				prepend_elem(OP_LIST, newGVREF(IVAL($1),$2), $3) );
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	FUNC '(' indirob expr ')'      /* print ($fh @args */
-			{ $$ = convert($1, OPf_STACKED,
-				prepend_elem(OP_LIST, newGVREF($1,$3), $4) ); }
+			{ $$ = convert(IVAL($1), OPf_STACKED,
+				prepend_elem(OP_LIST, newGVREF(IVAL($1),$3), $4) );
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 	|	term ARROW method '(' listexprcom ')' /* $foo->bar(list) */
 			{ $$ = convert(OP_ENTERSUB, OPf_STACKED,
 				append_elem(OP_LIST,
 				    prepend_elem(OP_LIST, scalar($1), $5),
-				    newUNOP(OP_METHOD, 0, $3))); }
+				    newUNOP(OP_METHOD, 0, $3)));
+			  TOKEN_GETMAD($2,$$,'A');
+			  TOKEN_GETMAD($4,$$,'(');
+			  TOKEN_GETMAD($6,$$,')');
+			}
 	|	term ARROW method                     /* $foo->bar */
 			{ $$ = convert(OP_ENTERSUB, OPf_STACKED,
 				append_elem(OP_LIST, scalar($1),
-				    newUNOP(OP_METHOD, 0, $3))); }
+				    newUNOP(OP_METHOD, 0, $3)));
+			  TOKEN_GETMAD($2,$$,'A');
+			}
 	|	METHOD indirob listexpr              /* new Class @args */
 			{ $$ = convert(OP_ENTERSUB, OPf_STACKED,
 				append_elem(OP_LIST,
 				    prepend_elem(OP_LIST, $2, $3),
-				    newUNOP(OP_METHOD, 0, $1))); }
+				    newUNOP(OP_METHOD, 0, $1)));
+			}
 	|	FUNCMETH indirob '(' listexprcom ')' /* method $object (@args) */
 			{ $$ = convert(OP_ENTERSUB, OPf_STACKED,
 				append_elem(OP_LIST,
 				    prepend_elem(OP_LIST, $2, $4),
-				    newUNOP(OP_METHOD, 0, $1))); }
+				    newUNOP(OP_METHOD, 0, $1)));
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 	|	LSTOP listexpr                       /* print @args */
-			{ $$ = convert($1, 0, $2); }
+			{ $$ = convert(IVAL($1), 0, $2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	FUNC '(' listexprcom ')'             /* print (@args) */
-			{ $$ = convert($1, 0, $3); }
+			{ $$ = convert(IVAL($1), 0, $3);
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	LSTOPSUB startanonsub block /* sub f(&@);   f { foo } ... */
 			{ SvREFCNT_inc(PL_compcv);
 			  $3 = newANONATTRSUB($2, 0, Nullop, $3); }
 		    listexpr		%prec LSTOP  /* ... @bar */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
 				 append_elem(OP_LIST,
-				   prepend_elem(OP_LIST, $3, $5), $1)); }
+				   prepend_elem(OP_LIST, $3, $5), $1));
+			}
 	;
 
 /* Names of methods. May use $object->$methodname */
@@ -490,128 +758,259 @@ subscripted:    star '{' expr ';' '}'        /* *main::{something} */
                         /* In this and all the hash accessors, ';' is
                          * provided by the tokeniser */
 			{ $$ = newBINOP(OP_GELEM, 0, $1, scalar($3));
-			    PL_expect = XOPERATOR; }
+			    PL_expect = XOPERATOR;
+			  TOKEN_GETMAD($2,$$,'{');
+			  TOKEN_GETMAD($4,$$,';');
+			  TOKEN_GETMAD($5,$$,'}');
+			}
 	|	scalar '[' expr ']'          /* $array[$element] */
-			{ $$ = newBINOP(OP_AELEM, 0, oopsAV($1), scalar($3)); }
+			{ $$ = newBINOP(OP_AELEM, 0, oopsAV($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'[');
+			  TOKEN_GETMAD($4,$$,']');
+			}
 	|	term ARROW '[' expr ']'      /* somearef->[$element] */
 			{ $$ = newBINOP(OP_AELEM, 0,
 					ref(newAVREF($1),OP_RV2AV),
-					scalar($4));}
+					scalar($4));
+			  TOKEN_GETMAD($2,$$,'a');
+			  TOKEN_GETMAD($3,$$,'[');
+			  TOKEN_GETMAD($5,$$,']');
+			}
 	|	subscripted '[' expr ']'    /* $foo->[$bar]->[$baz] */
 			{ $$ = newBINOP(OP_AELEM, 0,
 					ref(newAVREF($1),OP_RV2AV),
-					scalar($3));}
+					scalar($3));
+			  TOKEN_GETMAD($2,$$,'[');
+			  TOKEN_GETMAD($4,$$,']');
+			}
 	|	scalar '{' expr ';' '}'    /* $foo->{bar();} */
 			{ $$ = newBINOP(OP_HELEM, 0, oopsHV($1), jmaybe($3));
-			    PL_expect = XOPERATOR; }
+			    PL_expect = XOPERATOR;
+			  TOKEN_GETMAD($2,$$,'{');
+			  TOKEN_GETMAD($4,$$,';');
+			  TOKEN_GETMAD($5,$$,'}');
+			}
 	|	term ARROW '{' expr ';' '}' /* somehref->{bar();} */
 			{ $$ = newBINOP(OP_HELEM, 0,
 					ref(newHVREF($1),OP_RV2HV),
 					jmaybe($4));
-			    PL_expect = XOPERATOR; }
+			    PL_expect = XOPERATOR;
+			  TOKEN_GETMAD($2,$$,'a');
+			  TOKEN_GETMAD($3,$$,'{');
+			  TOKEN_GETMAD($5,$$,';');
+			  TOKEN_GETMAD($6,$$,'}');
+			}
 	|	subscripted '{' expr ';' '}' /* $foo->[bar]->{baz;} */
 			{ $$ = newBINOP(OP_HELEM, 0,
 					ref(newHVREF($1),OP_RV2HV),
 					jmaybe($3));
-			    PL_expect = XOPERATOR; }
+			    PL_expect = XOPERATOR;
+			  TOKEN_GETMAD($2,$$,'{');
+			  TOKEN_GETMAD($4,$$,';');
+			  TOKEN_GETMAD($5,$$,'}');
+			}
 	|	term ARROW '(' ')'          /* $subref->() */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-				   newCVREF(0, scalar($1))); }
+				   newCVREF(0, scalar($1)));
+			  TOKEN_GETMAD($2,$$,'a');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	term ARROW '(' expr ')'     /* $subref->(@args) */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
 				   append_elem(OP_LIST, $4,
-				       newCVREF(0, scalar($1)))); }
+				       newCVREF(0, scalar($1))));
+			  TOKEN_GETMAD($2,$$,'a');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 
 	|	subscripted '(' expr ')'   /* $foo->{bar}->(@args) */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
 				   append_elem(OP_LIST, $3,
-					       newCVREF(0, scalar($1)))); }
+					       newCVREF(0, scalar($1))));
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	subscripted '(' ')'        /* $foo->{bar}->() */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-				   newCVREF(0, scalar($1))); }
+				   newCVREF(0, scalar($1)));
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	'(' expr ')' '[' expr ']'            /* list slice */
-			{ $$ = newSLICEOP(0, $5, $2); }
+			{ $$ = newSLICEOP(0, $5, $2);
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			  TOKEN_GETMAD($4,$$,'[');
+			  TOKEN_GETMAD($6,$$,']');
+			}
 	|	'(' ')' '[' expr ']'                 /* empty list slice! */
-			{ $$ = newSLICEOP(0, $4, Nullop); }
+			{ $$ = newSLICEOP(0, $4, Nullop);
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($2,$$,')');
+			  TOKEN_GETMAD($3,$$,'[');
+			  TOKEN_GETMAD($5,$$,']');
+			}
     ;
 
 /* Binary operators between terms */
-termbinop	:	term ASSIGNOP term             /* $x = $y */
-			{ $$ = newASSIGNOP(OPf_STACKED, $1, $2, $3); }
+termbinop:	term ASSIGNOP term                     /* $x = $y */
+			{ $$ = newASSIGNOP(OPf_STACKED, $1, IVAL($2), $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term POWOP term                        /* $x ** $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term MULOP term                        /* $x * $y, $x x $y */
-			{   if ($2 != OP_REPEAT)
+			{   if (IVAL($2) != OP_REPEAT)
 				scalar($1);
-			    $$ = newBINOP($2, 0, $1, scalar($3)); }
+			    $$ = newBINOP(IVAL($2), 0, $1, scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term ADDOP term                        /* $x + $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term SHIFTOP term                      /* $x >> $y, $x << $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term RELOP term                        /* $x > $y, etc. */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term EQOP term                         /* $x == $y, $x eq $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term BITANDOP term                     /* $x & $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term BITOROP term                      /* $x | $y */
-			{ $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+			{ $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term DOTDOT term                       /* $x..$y, $x...$y */
-			{ $$ = newRANGE($2, scalar($1), scalar($3));}
+			{
+			  $$ = newRANGE(IVAL($2), scalar($1), scalar($3));
+			  DO_MAD(
+			      UNOP *op;
+			      op = (UNOP*)$$;
+			      op = (UNOP*)op->op_first;	/* get to flop */
+			      op = (UNOP*)op->op_first;	/* get to flip */
+			      op = (UNOP*)op->op_first;	/* get to range */
+			      token_getmad($2,(OP*)op,'o');
+			    )
+			}
 	|	term ANDAND term                       /* $x && $y */
-			{ $$ = newLOGOP(OP_AND, 0, $1, $3); }
+			{ $$ = newLOGOP(OP_AND, 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term OROR term                         /* $x || $y */
-			{ $$ = newLOGOP(OP_OR, 0, $1, $3); }
+			{ $$ = newLOGOP(OP_OR, 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term DORDOR term                       /* $x // $y */
-			{ $$ = newLOGOP(OP_DOR, 0, $1, $3); }
+			{ $$ = newLOGOP(OP_DOR, 0, $1, $3);
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term MATCHOP term                      /* $x =~ /$y/ */
-			{ $$ = bind_match($2, $1, $3); }
+			{ $$ = bind_match(IVAL($2), $1, $3);
+			  TOKEN_GETMAD($2,
+				($$->op_type == OP_NOT
+				    ? ((UNOP*)$$)->op_first : $$),
+				'~');
+			}
     ;
 
 /* Unary operators and terms */
 termunop : '-' term %prec UMINUS                       /* -$x */
-			{ $$ = newUNOP(OP_NEGATE, 0, scalar($2)); }
+			{ $$ = newUNOP(OP_NEGATE, 0, scalar($2));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	'+' term %prec UMINUS                  /* +$x */
-			{ $$ = $2; }
+			{ $$ = IF_MAD(
+				    newUNOP(OP_NULL, 0, $2),
+				    $2
+				);
+			  TOKEN_GETMAD($1,$$,'+');
+			}
 	|	'!' term                               /* !$x */
-			{ $$ = newUNOP(OP_NOT, 0, scalar($2)); }
+			{ $$ = newUNOP(OP_NOT, 0, scalar($2));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	'~' term                               /* ~$x */
-			{ $$ = newUNOP(OP_COMPLEMENT, 0, scalar($2));}
+			{ $$ = newUNOP(OP_COMPLEMENT, 0, scalar($2));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	term POSTINC                           /* $x++ */
 			{ $$ = newUNOP(OP_POSTINC, 0,
-					mod(scalar($1), OP_POSTINC)); }
+					mod(scalar($1), OP_POSTINC));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	term POSTDEC                           /* $x-- */
 			{ $$ = newUNOP(OP_POSTDEC, 0,
-					mod(scalar($1), OP_POSTDEC)); }
+					mod(scalar($1), OP_POSTDEC));
+			  TOKEN_GETMAD($2,$$,'o');
+			}
 	|	PREINC term                            /* ++$x */
 			{ $$ = newUNOP(OP_PREINC, 0,
-					mod(scalar($2), OP_PREINC)); }
+					mod(scalar($2), OP_PREINC));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	PREDEC term                            /* --$x */
 			{ $$ = newUNOP(OP_PREDEC, 0,
-					mod(scalar($2), OP_PREDEC)); }
+					mod(scalar($2), OP_PREDEC));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 
     ;
 
 /* Constructors for anonymous data */
 anonymous:	'[' expr ']'
-			{ $$ = newANONLIST($2); }
+			{ $$ = newANONLIST($2);
+			  TOKEN_GETMAD($1,$$,'[');
+			  TOKEN_GETMAD($3,$$,']');
+			}
 	|	'[' ']'
-			{ $$ = newANONLIST(Nullop); }
+			{ $$ = newANONLIST(Nullop);
+			  TOKEN_GETMAD($1,$$,'[');
+			  TOKEN_GETMAD($2,$$,']');
+			}
 	|	HASHBRACK expr ';' '}'	%prec '(' /* { foo => "Bar" } */
-			{ $$ = newANONHASH($2); }
+			{ $$ = newANONHASH($2);
+			  TOKEN_GETMAD($1,$$,'{');
+			  TOKEN_GETMAD($3,$$,';');
+			  TOKEN_GETMAD($4,$$,'}');
+			}
 	|	HASHBRACK ';' '}'	%prec '(' /* { } (';' by tokener) */
-			{ $$ = newANONHASH(Nullop); }
+			{ $$ = newANONHASH(Nullop);
+			  TOKEN_GETMAD($1,$$,'{');
+			  TOKEN_GETMAD($2,$$,';');
+			  TOKEN_GETMAD($3,$$,'}');
+			}
 	|	ANONSUB startanonsub proto subattrlist block	%prec '('
 			{ SvREFCNT_inc(PL_compcv);
-			  $$ = newANONATTRSUB($2, $3, $4, $5); }
+			  $$ = newANONATTRSUB($2, $3, $4, $5);
+			  TOKEN_GETMAD($1,$$,'o');
+			  OP_GETMAD($3,$$,'s');
+			  OP_GETMAD($4,$$,'a');
+			}
 
     ;
 
 /* Things called with "do" */
 termdo	:       DO term	%prec UNIOP                     /* do $filename */
-			{ $$ = dofile($2, $1); }
+			{ $$ = dofile($2, $1);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	DO block	%prec '('               /* do { code */
-			{ $$ = newUNOP(OP_NULL, OPf_SPECIAL, scope($2)); }
+			{ $$ = newUNOP(OP_NULL, OPf_SPECIAL, scope($2));
+			  TOKEN_GETMAD($1,$$,'D');
+			}
 	|	DO WORD '(' ')'                         /* do somesub() */
 			{ $$ = newUNOP(OP_ENTERSUB,
 			    OPf_SPECIAL|OPf_STACKED,
@@ -619,7 +1018,11 @@ termdo	:       DO term	%prec UNIOP                     /* do $filename */
 				scalar(newCVREF(
 				    (OPpENTERSUB_AMPER<<8),
 				    scalar($2)
-				)),Nullop)); dep();}
+				)),Nullop)); dep();
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	DO WORD '(' expr ')'                    /* do somesub(@args) */
 			{ $$ = newUNOP(OP_ENTERSUB,
 			    OPf_SPECIAL|OPf_STACKED,
@@ -628,16 +1031,28 @@ termdo	:       DO term	%prec UNIOP                     /* do $filename */
 				scalar(newCVREF(
 				    (OPpENTERSUB_AMPER<<8),
 				    scalar($2)
-				)))); dep();}
+				)))); dep();
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 	|	DO scalar '(' ')'                      /* do $subref () */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_SPECIAL|OPf_STACKED,
 			    prepend_elem(OP_LIST,
-				scalar(newCVREF(0,scalar($2))), Nullop)); dep();}
+				scalar(newCVREF(0,scalar($2))), Nullop)); dep();
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	DO scalar '(' expr ')'                 /* do $subref (@args) */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_SPECIAL|OPf_STACKED,
 			    prepend_elem(OP_LIST,
 				$4,
-				scalar(newCVREF(0,scalar($2))))); dep();}
+				scalar(newCVREF(0,scalar($2))))); dep();
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($3,$$,'(');
+			  TOKEN_GETMAD($5,$$,')');
+			}
 
         ;
 
@@ -646,17 +1061,30 @@ term	:	termbinop
 	|	anonymous
 	|	termdo
 	|	term '?' term ':' term
-			{ $$ = newCONDOP(0, $1, $3, $5); }
+			{ $$ = newCONDOP(0, $1, $3, $5);
+			  TOKEN_GETMAD($2,$$,'?');
+			  TOKEN_GETMAD($4,$$,':');
+			}
 	|	REFGEN term                          /* \$x, \@y, \%z */
-			{ $$ = newUNOP(OP_REFGEN, 0, mod($2,OP_REFGEN)); }
+			{ $$ = newUNOP(OP_REFGEN, 0, mod($2,OP_REFGEN));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	myattrterm	%prec UNIOP
 			{ $$ = $1; }
 	|	LOCAL term	%prec UNIOP
-			{ $$ = localize($2,$1); }
+			{ $$ = localize($2,IVAL($1));
+			  TOKEN_GETMAD($1,$$,'d');
+			}
 	|	'(' expr ')'
-			{ $$ = sawparens($2); }
+			{ $$ = sawparens(IF_MAD(newUNOP(OP_NULL,0,$2), $2));
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	'(' ')'
-			{ $$ = sawparens(newNULLLIST()); }
+			{ $$ = sawparens(newNULLLIST());
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($2,$$,')');
+			}
 	|	scalar	%prec '('
 			{ $$ = $1; }
 	|	star	%prec '('
@@ -674,42 +1102,76 @@ term	:	termbinop
 				newOP(OP_PUSHMARK, 0),
 				    newLISTOP(OP_ASLICE, 0,
 					list($3),
-					ref($1, OP_ASLICE))); }
+					ref($1, OP_ASLICE)));
+			  TOKEN_GETMAD($2,$$,'[');
+			  TOKEN_GETMAD($4,$$,']');
+			}
 	|	ary '{' expr ';' '}'                 /* @hash{@keys} */
 			{ $$ = prepend_elem(OP_HSLICE,
 				newOP(OP_PUSHMARK, 0),
 				    newLISTOP(OP_HSLICE, 0,
 					list($3),
 					ref(oopsHV($1), OP_HSLICE)));
-			    PL_expect = XOPERATOR; }
+			    PL_expect = XOPERATOR;
+			  TOKEN_GETMAD($2,$$,'{');
+			  TOKEN_GETMAD($4,$$,';');
+			  TOKEN_GETMAD($5,$$,'}');
+			}
 	|	THING	%prec '('
 			{ $$ = $1; }
 	|	amper                                /* &foo; */
 			{ $$ = newUNOP(OP_ENTERSUB, 0, scalar($1)); }
 	|	amper '(' ')'                        /* &foo() */
-			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1)); }
+			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1));
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	amper '(' expr ')'                   /* &foo(@args) */
-			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-			    append_elem(OP_LIST, $3, scalar($1))); }
+			{
+			  $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
+				append_elem(OP_LIST, $3, scalar($1)));
+			  DO_MAD(
+			      OP* op = $$;
+			      if (op->op_type == OP_CONST) { /* defeat const fold */
+				op = (OP*)op->op_madprop->mad_val;
+			      }
+			      token_getmad($2,op,'(');
+			      token_getmad($4,op,')');
+			  )
+			}
 	|	NOAMP WORD listexpr                  /* foo(@args) */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-			    append_elem(OP_LIST, $3, scalar($2))); }
+			    append_elem(OP_LIST, $3, scalar($2)));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	LOOPEX  /* loop exiting command (goto, last, dump, etc) */
-			{ $$ = newOP($1, OPf_SPECIAL);
-			    PL_hints |= HINT_BLOCK_SCOPE; }
+			{ $$ = newOP(IVAL($1), OPf_SPECIAL);
+			    PL_hints |= HINT_BLOCK_SCOPE;
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	LOOPEX term
-			{ $$ = newLOOPEX($1,$2); }
+			{ $$ = newLOOPEX(IVAL($1),$2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	NOTOP argexpr                        /* not $foo */
-			{ $$ = newUNOP(OP_NOT, 0, scalar($2)); }
+			{ $$ = newUNOP(OP_NOT, 0, scalar($2));
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	UNIOP                                /* Unary op, $_ implied */
-			{ $$ = newOP($1, 0); }
-	|	UNIOP block                          /* eval { foo } */
-			{ $$ = newUNOP($1, 0, $2); }
+			{ $$ = newOP(IVAL($1), 0);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
+	|	UNIOP block                          /* eval { foo }* */
+			{ $$ = newUNOP(IVAL($1), 0, $2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	UNIOP term                           /* Unary op */
-			{ $$ = newUNOP($1, 0, $2); }
-	|	REQUIRE                              /* require, $_ implied */
+			{ $$ = newUNOP(IVAL($1), 0, $2);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
+	|	REQUIRE                              /* require, $_ implied *//* FIMXE for MAD needed? */
 			{ $$ = newOP(OP_REQUIRE, $1 ? OPf_SPECIAL : 0); }
-	|	REQUIRE term                         /* require Foo */
+	|	REQUIRE term                         /* require Foo *//* FIMXE for MAD needed? */
 			{ $$ = newUNOP(OP_REQUIRE, $1 ? OPf_SPECIAL : 0, $2); }
 	|	UNIOPSUB
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1)); }
@@ -717,35 +1179,68 @@ term	:	termbinop
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
 			    append_elem(OP_LIST, $2, scalar($1))); }
 	|	FUNC0                                /* Nullary operator */
-			{ $$ = newOP($1, 0); }
+			{ $$ = newOP(IVAL($1), 0);
+			  TOKEN_GETMAD($1,$$,'o');
+			}
 	|	FUNC0 '(' ')'
-			{ $$ = newOP($1, 0); }
+			{ $$ = newOP(IVAL($1), 0);
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	FUNC0SUB                             /* Sub treated as nullop */
 			{ $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
 				scalar($1)); }
 	|	FUNC1 '(' ')'                        /* not () */
-			{ $$ = $1 == OP_NOT ? newUNOP($1, 0, newSVOP(OP_CONST, 0, newSViv(0)))
-					    : newOP($1, OPf_SPECIAL); }
+			{ $$ = (IVAL($1) == OP_NOT)
+			    ? newUNOP(IVAL($1), 0, newSVOP(OP_CONST, 0, newSViv(0)))
+			    : newOP(IVAL($1), OPf_SPECIAL);
+
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	FUNC1 '(' expr ')'                   /* not($foo) */
-			{ $$ = newUNOP($1, 0, $3); }
-	|	PMFUNC '(' argexpr ')'           /* m//, s///, tr/// */
-			{ $$ = pmruntime($1, $3, 1); }
+			{ $$ = newUNOP(IVAL($1), 0, $3);
+			  TOKEN_GETMAD($1,$$,'o');
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
+	|	PMFUNC '(' argexpr ')'		/* m//, s///, tr/// */
+			{ $$ = pmruntime($1, $3, 1);
+			  TOKEN_GETMAD($2,$$,'(');
+			  TOKEN_GETMAD($4,$$,')');
+			}
 	|	WORD
 	|	listop
 	;
 
 /* "my" declarations, with optional attributes */
 myattrterm:	MY myterm myattrlist
-			{ $$ = my_attrs($2,$3); }
+			{ $$ = my_attrs($2,$3);
+			  DO_MAD(
+			      token_getmad($1,$$,'d');
+			      append_madprops($3->op_madprop, $$, 'a');
+			      $3->op_madprop = 0;
+			  )
+			}
 	|	MY myterm
-			{ $$ = localize($2,$1); }
+			{ $$ = localize($2,IVAL($1));
+			  TOKEN_GETMAD($1,$$,'d');
+			}
 	;
 
 /* Things that can be "my"'d */
 myterm	:	'(' expr ')'
-			{ $$ = sawparens($2); }
+			{ $$ = sawparens($2);
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($3,$$,')');
+			}
 	|	'(' ')'
-			{ $$ = sawparens(newNULLLIST()); }
+			{ $$ = sawparens(newNULLLIST());
+			  TOKEN_GETMAD($1,$$,'(');
+			  TOKEN_GETMAD($2,$$,')');
+			}
 	|	scalar	%prec '('
 			{ $$ = $1; }
 	|	hsh 	%prec '('
@@ -766,7 +1261,16 @@ listexprcom:	/* NULL */
 	|	expr
 			{ $$ = $1; }
 	|	expr ','
-			{ $$ = $1; }
+			{
+#ifdef MAD
+			  OP* op = newNULLLIST();
+			  token_getmad($2,op,',');
+			  $$ = append_elem(OP_LIST, $1, op);
+#else
+			  $$ = $1;
+#endif
+
+			}
 	;
 
 /* A little bit of trickery to make "for my $foo (@bar)" actually be
@@ -776,34 +1280,46 @@ my_scalar:	scalar
 	;
 
 amper	:	'&' indirob
-			{ $$ = newCVREF($1,$2); }
+			{ $$ = newCVREF(IVAL($1),$2);
+			  TOKEN_GETMAD($1,$$,'&');
+			}
 	;
 
 scalar	:	'$' indirob
-			{ $$ = newSVREF($2); }
+			{ $$ = newSVREF($2);
+			  TOKEN_GETMAD($1,$$,'$');
+			}
 	;
 
 ary	:	'@' indirob
-			{ $$ = newAVREF($2); }
+			{ $$ = newAVREF($2);
+			  TOKEN_GETMAD($1,$$,'@');
+			}
 	;
 
 hsh	:	'%' indirob
-			{ $$ = newHVREF($2); }
+			{ $$ = newHVREF($2);
+			  TOKEN_GETMAD($1,$$,'%');
+			}
 	;
 
 arylen	:	DOLSHARP indirob
-			{ $$ = newAVREF($2); }
+			{ $$ = newAVREF($2);
+			  TOKEN_GETMAD($1,$$,'l');
+			}
 	;
 
 star	:	'*' indirob
-			{ $$ = newGVREF(0,$2); }
+			{ $$ = newGVREF(0,$2);
+			  TOKEN_GETMAD($1,$$,'*');
+			}
 	;
 
 /* Indirect objects */
 indirob	:	WORD
 			{ $$ = scalar($1); }
 	|	scalar %prec PREC_LOW
-			{ $$ = scalar($1);  }
+			{ $$ = scalar($1); }
 	|	block
 			{ $$ = scope($1); }