* c-lex.c (cb_def_pragma): Update.

	(c_lex): Update, and skip padding.
	* cppexp.c (lex, parse_defined): Update, remove unused variable.
	* cpphash.h (struct toklist): Delete.
	(union utoken): New.
	(struct cpp_context): Update.
	(struct cpp_reader): New members eof, avoid_paste.
	(_cpp_temp_token): New.
	* cppinit.c (cpp_create_reader): Update.
	* cpplex.c (_cpp_temp_token): New.
	(_cpp_lex_direct): Add PREV_WHITE when parsing args.
	(cpp_output_token): Don't print leading whitespace.
	(cpp_output_line): Update.
	* cpplib.c (glue_header_name, parse_include, get__Pragma_string,
	do_include_common, do_line, do_ident, do_pragma,
	do_pragma_dependency, _cpp_do__Pragma, parse_answer,
	parse_assertion): Update.
	(get_token_no_padding): New.
	* cpplib.h (CPP_PADDING): New.
	(AVOID_LPASTE): Delete.
	(struct cpp_token): New union member source.
	(cpp_get_token): Update.
	* cppmacro.c (macro_arg): Convert to use pointers to const tokens.
	(builtin_macro, paste_all_tokens, paste_tokens, funlike_invocation_p,
	replace_args, quote_string, stringify_arg, parse_arg, next_context,
	enter_macro_context, expand_arg, _cpp_pop_context, cpp_scan_nooutput,
	_cpp_backup_tokens, _cpp_create_definition): Update.
	(push_arg_context): Delete.
	(padding_token, push_token_context, push_ptoken_context): New.
	(make_string_token, make_number_token): Update, rename.
	(cpp_get_token): Update to handle tokens as pointers to const,
	and insert padding appropriately.
	* cppmain.c (struct printer): New member prev.
	(check_multiline_token): Constify.
	(do_preprocessing, cb_line_change): Update.
	(scan_translation_unit): Update to handle spacing.
	* scan-decls.c (get_a_token): New.
	(skip_to_closing_brace, scan_decls): Update.
	* fix-header.c (read_scan_file): Update.

	* doc/cpp.texi: Update.

	* gcc.dg/cpp/macro10.c: New test.
	* gcc.dg/cpp/strify3.c: New test.
	* gcc.dg/cpp/spacing1.c: Add tests.
	* gcc.dg/cpp/19990703-1.c: Remove bogus test.
	* gcc.dg/cpp/20000625-2.c: Fudge to pass.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@45793 138bc75d-0d04-0410-961f-82ee72b054a4

19 files changed, 794 insertions, 566 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 66792a3df58..2ef183d2b35 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,47 @@
+2001-09-24  Neil Booth  <neil@daikokuya.demon.co.uk>
+
+	* c-lex.c (cb_def_pragma): Update.
+	(c_lex): Update, and skip padding.
+	* cppexp.c (lex, parse_defined): Update, remove unused variable.
+	* cpphash.h (struct toklist): Delete.
+	(union utoken): New.
+	(struct cpp_context): Update.
+	(struct cpp_reader): New members eof, avoid_paste.
+	(_cpp_temp_token): New.
+	* cppinit.c (cpp_create_reader): Update.
+	* cpplex.c (_cpp_temp_token): New.
+	(_cpp_lex_direct): Add PREV_WHITE when parsing args.
+	(cpp_output_token): Don't print leading whitespace.
+	(cpp_output_line): Update.
+	* cpplib.c (glue_header_name, parse_include, get__Pragma_string,
+	do_include_common, do_line, do_ident, do_pragma,
+	do_pragma_dependency, _cpp_do__Pragma, parse_answer,
+	parse_assertion): Update.
+	(get_token_no_padding): New.
+	* cpplib.h (CPP_PADDING): New.
+	(AVOID_LPASTE): Delete.
+	(struct cpp_token): New union member source.
+	(cpp_get_token): Update.
+	* cppmacro.c (macro_arg): Convert to use pointers to const tokens.
+	(builtin_macro, paste_all_tokens, paste_tokens, funlike_invocation_p,
+	replace_args, quote_string, stringify_arg, parse_arg, next_context,
+	enter_macro_context, expand_arg, _cpp_pop_context, cpp_scan_nooutput,
+	_cpp_backup_tokens, _cpp_create_definition): Update.
+	(push_arg_context): Delete.
+	(padding_token, push_token_context, push_ptoken_context): New.
+	(make_string_token, make_number_token): Update, rename.
+	(cpp_get_token): Update to handle tokens as pointers to const,
+	and insert padding appropriately.
+	* cppmain.c (struct printer): New member prev.
+	(check_multiline_token): Constify.
+	(do_preprocessing, cb_line_change): Update.
+	(scan_translation_unit): Update to handle spacing.
+	* scan-decls.c (get_a_token): New.
+	(skip_to_closing_brace, scan_decls): Update.
+	* fix-header.c (read_scan_file): Update.
+
+	* doc/cpp.texi: Update.
+
 2001-09-24  Kaveh R. Ghazi  <ghazi@caip.rutgers.edu>
 
 	* c-aux-info.c (affix_data_type): Use ATTRIBUTE_MALLOC.  Avoid
diff --git a/gcc/c-lex.c b/gcc/c-lex.c
index 3a80a74a550..8e6582f06db 100644
--- a/gcc/c-lex.c
+++ b/gcc/c-lex.c
@@ -335,13 +335,13 @@ cb_def_pragma (pfile, line)
   if (warn_unknown_pragmas > in_system_header)
     {
       const unsigned char *space, *name = 0;
-      cpp_token s;
+      const cpp_token *s;
 
-      cpp_get_token (pfile, &s);
-      space = cpp_token_as_text (pfile, &s);
-      cpp_get_token (pfile, &s);
-      if (s.type == CPP_NAME)
-	name = cpp_token_as_text (pfile, &s);
+      s = cpp_get_token (pfile);
+      space = cpp_token_as_text (pfile, s);
+      s = cpp_get_token (pfile);
+      if (s->type == CPP_NAME)
+	name = cpp_token_as_text (pfile, s);
 
       lineno = SOURCE_LINE (map, line);
       if (name)
@@ -767,12 +767,13 @@ int
 c_lex (value)
      tree *value;
 {
-  cpp_token tok;
-  enum cpp_ttype type;
+  const cpp_token *tok;
 
   retry:
   timevar_push (TV_CPP);
-  cpp_get_token (parse_in, &tok);
+  do
+    tok = cpp_get_token (parse_in);
+  while (tok->type == CPP_PADDING);
   timevar_pop (TV_CPP);
 
   /* The C++ front end does horrible things with the current line
@@ -781,37 +782,36 @@ c_lex (value)
   lineno = src_lineno;
 
   *value = NULL_TREE;
-  type = tok.type;
-  switch (type)
+  switch (tok->type)
     {
     case CPP_OPEN_BRACE:  indent_level++;  break;
     case CPP_CLOSE_BRACE: indent_level--;  break;
 
-    /* Issue this error here, where we can get at tok.val.c.  */
+    /* Issue this error here, where we can get at tok->val.c.  */
     case CPP_OTHER:
-      if (ISGRAPH (tok.val.c))
-	error ("stray '%c' in program", tok.val.c);
+      if (ISGRAPH (tok->val.c))
+	error ("stray '%c' in program", tok->val.c);
       else
-	error ("stray '\\%o' in program", tok.val.c);
+	error ("stray '\\%o' in program", tok->val.c);
       goto retry;
       
     case CPP_NAME:
-      *value = HT_IDENT_TO_GCC_IDENT (HT_NODE (tok.val.node));
+      *value = HT_IDENT_TO_GCC_IDENT (HT_NODE (tok->val.node));
       break;
 
     case CPP_NUMBER:
-      *value = lex_number ((const char *)tok.val.str.text, tok.val.str.len);
+      *value = lex_number ((const char *)tok->val.str.text, tok->val.str.len);
       break;
 
     case CPP_CHAR:
     case CPP_WCHAR:
-      *value = lex_charconst (&tok);
+      *value = lex_charconst (tok);
       break;
 
     case CPP_STRING:
     case CPP_WSTRING:
-      *value = lex_string ((const char *)tok.val.str.text,
-			   tok.val.str.len, tok.type == CPP_WSTRING);
+      *value = lex_string ((const char *)tok->val.str.text,
+			   tok->val.str.len, tok->type == CPP_WSTRING);
       break;
 
       /* These tokens should not be visible outside cpplib.  */
@@ -823,7 +823,7 @@ c_lex (value)
     default: break;
     }
 
-  return type;
+  return tok->type;
 }
 
 #define ERROR(msgid) do { error(msgid); goto syntax_error; } while(0)
diff --git a/gcc/cppexp.c b/gcc/cppexp.c
index bdc09ed8985..c79f20d2be9 100644
--- a/gcc/cppexp.c
+++ b/gcc/cppexp.c
@@ -36,7 +36,7 @@ static HOST_WIDEST_INT right_shift PARAMS ((cpp_reader *, HOST_WIDEST_INT,
 					    unsigned HOST_WIDEST_INT));
 static struct op parse_number PARAMS ((cpp_reader *, const cpp_token *));
 static struct op parse_defined PARAMS ((cpp_reader *));
-static struct op lex PARAMS ((cpp_reader *, int, cpp_token *));
+static struct op lex PARAMS ((cpp_reader *, int));
 static const unsigned char *op_as_text PARAMS ((cpp_reader *, enum cpp_ttype));
 
 struct op
@@ -217,44 +217,40 @@ parse_defined (pfile)
 {
   int paren = 0;
   cpp_hashnode *node = 0;
-  cpp_token token;
+  const cpp_token *token;
   struct op op;
 
   /* Don't expand macros.  */
   pfile->state.prevent_expansion++;
 
-  cpp_get_token (pfile, &token);
-  if (token.type == CPP_OPEN_PAREN)
+  token = cpp_get_token (pfile);
+  if (token->type == CPP_OPEN_PAREN)
     {
       paren = 1;
-      cpp_get_token (pfile, &token);
+      token = cpp_get_token (pfile);
     }
 
-  if (token.type == CPP_NAME)
+  if (token->type == CPP_NAME)
     {
-      node = token.val.node;
-      if (paren)
+      node = token->val.node;
+      if (paren && cpp_get_token (pfile)->type != CPP_CLOSE_PAREN)
 	{
-	  cpp_get_token (pfile, &token);
-	  if (token.type != CPP_CLOSE_PAREN)
-	    {
-	      cpp_error (pfile, "missing ')' after \"defined\"");
-	      node = 0;
-	    }
+	  cpp_error (pfile, "missing ')' after \"defined\"");
+	  node = 0;
 	}
     }
   else
     {
       cpp_error (pfile, "operator \"defined\" requires an identifier");
-      if (token.flags & NAMED_OP)
+      if (token->flags & NAMED_OP)
 	{
 	  cpp_token op;
 
 	  op.flags = 0;
-	  op.type = token.type;
+	  op.type = token->type;
 	  cpp_error (pfile,
 		     "(\"%s\" is an alternative token for \"%s\" in C++)",
-		     cpp_token_as_text (pfile, &token),
+		     cpp_token_as_text (pfile, token),
 		     cpp_token_as_text (pfile, &op));
 	}
     }
@@ -282,14 +278,12 @@ parse_defined (pfile)
    CPP_EOF, or the type of an operator token.  */
 
 static struct op
-lex (pfile, skip_evaluation, token)
+lex (pfile, skip_evaluation)
      cpp_reader *pfile;
      int skip_evaluation;
-     cpp_token *token;
 {
   struct op op;
-
-  cpp_get_token (pfile, token);
+  const cpp_token *token = cpp_get_token (pfile);
 
   switch (token->type)
     {
@@ -578,7 +572,6 @@ _cpp_parse_expr (pfile)
   struct op init_stack[INIT_STACK_SIZE];
   struct op *stack = init_stack;
   struct op *limit = stack + INIT_STACK_SIZE;
-  cpp_token token;
   register struct op *top = stack + 1;
   int skip_evaluation = 0;
   int result;
@@ -603,7 +596,7 @@ _cpp_parse_expr (pfile)
       struct op op;
 
       /* Read a token */
-      op = lex (pfile, skip_evaluation, &token);
+      op = lex (pfile, skip_evaluation);
       lex_count++;
 
       /* If the token is an operand, push its value and get next
diff --git a/gcc/cpphash.h b/gcc/cpphash.h
index 64deaa2fa23..001ac60c53e 100644
--- a/gcc/cpphash.h
+++ b/gcc/cpphash.h
@@ -95,11 +95,10 @@ struct search_path
 /* #include types.  */
 enum include_type {IT_INCLUDE, IT_INCLUDE_NEXT, IT_IMPORT, IT_CMDLINE};
 
-typedef struct toklist toklist;
-struct toklist
+union utoken
 {
-  cpp_token *first;
-  cpp_token *limit;
+  const cpp_token *token;
+  const cpp_token **ptoken;
 };
 
 typedef struct tokenrun tokenrun;
@@ -117,10 +116,14 @@ struct cpp_context
 
   /* Contexts other than the base context are contiguous tokens.
      e.g. macro expansions, expanded argument tokens.  */
-  struct toklist list;
+  union utoken first;
+  union utoken last;
 
   /* For a macro context, these are the macro and its arguments.  */
   cpp_macro *macro;
+
+  /* True if utoken element is token, else ptoken.  */
+  bool direct_p;
 };
 
 struct lexer_state
@@ -294,6 +297,10 @@ struct cpp_reader
   cpp_token date;
   cpp_token time;
 
+  /* EOF token, and a token forcing paste avoidance.  */
+  cpp_token avoid_paste;
+  cpp_token eof;
+
   /* Opaque handle to the dependencies of mkdeps.c.  Used by -M etc.  */
   struct deps *deps;
 
@@ -398,6 +405,7 @@ extern void _cpp_pop_file_buffer	PARAMS ((cpp_reader *,
 extern int _cpp_parse_expr		PARAMS ((cpp_reader *));
 
 /* In cpplex.c */
+extern cpp_token *_cpp_temp_token	PARAMS ((cpp_reader *));
 extern const cpp_token *_cpp_lex_token	PARAMS ((cpp_reader *));
 extern cpp_token *_cpp_lex_direct	PARAMS ((cpp_reader *));
 extern int _cpp_equiv_tokens		PARAMS ((const cpp_token *,
diff --git a/gcc/cppinit.c b/gcc/cppinit.c
index 7c7f8dcdf05..11c86aa3377 100644
--- a/gcc/cppinit.c
+++ b/gcc/cppinit.c
@@ -510,8 +510,12 @@ cpp_create_reader (table, lang)
   /* Initialize lexer state.  */
   pfile->state.save_comments = ! CPP_OPTION (pfile, discard_comments);
 
-  /* Indicate date and time not yet calculated.  */
+  /* Set up static tokens.  */
   pfile->date.type = CPP_EOF;
+  pfile->avoid_paste.type = CPP_PADDING;
+  pfile->avoid_paste.val.source = NULL;
+  pfile->eof.type = CPP_EOF;
+  pfile->eof.flags = 0;
 
   /* Create a token buffer for the lexer.  */
   _cpp_init_tokenrun (&pfile->base_run, 250);
diff --git a/gcc/cpplex.c b/gcc/cpplex.c
index e734f4021b2..e822ba57719 100644
--- a/gcc/cpplex.c
+++ b/gcc/cpplex.c
@@ -931,6 +931,29 @@ next_tokenrun (run)
   return run->next;
 }
 
+/* Allocate a single token that is invalidated at the same time as the
+   rest of the tokens on the line.  Has its line and col set to the
+   same as the last lexed token, so that diagnostics appear in the
+   right place.  */
+cpp_token *
+_cpp_temp_token (pfile)
+     cpp_reader *pfile;
+{
+  cpp_token *old, *result;
+
+  old = pfile->cur_token - 1;
+  if (pfile->cur_token == pfile->cur_run->limit)
+    {
+      pfile->cur_run = next_tokenrun (pfile->cur_run);
+      pfile->cur_token = pfile->cur_run->base;
+    }
+
+  result = pfile->cur_token++;
+  result->line = old->line;
+  result->col = old->col;
+  return result;
+}
+
 /* Lex a token into RESULT (external interface).  Takes care of issues
    like directive handling, token lookahead, multiple include
    opimisation and skipping.  */
@@ -1057,6 +1080,8 @@ _cpp_lex_direct (pfile)
       buffer->saved_flags = BOL;
       if (! pfile->state.in_directive)
 	{
+	  if (pfile->state.parsing_args == 2)
+	    buffer->saved_flags |= PREV_WHITE;
 	  if (!pfile->keep_tokens)
 	    {
 	      pfile->cur_run = &pfile->base_run;
@@ -1476,17 +1501,14 @@ cpp_type2name (type)
   return (const char *) token_spellings[type].name;
 }
 
-/* Writes the spelling of token to FP.  Separate from cpp_spell_token
-   for efficiency - to avoid double-buffering.  Also, outputs a space
-   if PREV_WHITE is flagged.  */
+/* Writes the spelling of token to FP, without any preceding space.
+   Separated from cpp_spell_token for efficiency - to avoid stdio
+   double-buffering.  */
 void
 cpp_output_token (token, fp)
      const cpp_token *token;
      FILE *fp;
 {
-  if (token->flags & PREV_WHITE)
-    putc (' ', fp);
-
   switch (TOKEN_SPELL (token))
     {
     case SPELL_OPERATOR:
@@ -1729,20 +1751,22 @@ cpp_avoid_paste (pfile, token1, token2)
 }
 
 /* Output all the remaining tokens on the current line, and a newline
-   character, to FP.  Leading whitespace is removed.  */
+   character, to FP.  Leading whitespace is removed.  If there are
+   macros, special token padding is not performed.  */
 void
 cpp_output_line (pfile, fp)
      cpp_reader *pfile;
      FILE *fp;
 {
-  cpp_token token;
+  const cpp_token *token;
 
-  cpp_get_token (pfile, &token);
-  token.flags &= ~PREV_WHITE;
-  while (token.type != CPP_EOF)
+  token = cpp_get_token (pfile);
+  while (token->type != CPP_EOF)
     {
-      cpp_output_token (&token, fp);
-      cpp_get_token (pfile, &token);
+      cpp_output_token (token, fp);
+      token = cpp_get_token (pfile);
+      if (token->flags & PREV_WHITE)
+	putc (' ', fp);
     }
 
   putc ('\n', fp);
diff --git a/gcc/cpplib.c b/gcc/cpplib.c
index 63d0c174a83..d164b4e2492 100644
--- a/gcc/cpplib.c
+++ b/gcc/cpplib.c
@@ -86,8 +86,8 @@ static void directive_diagnostics
 	PARAMS ((cpp_reader *, const directive *, int));
 static void run_directive	PARAMS ((cpp_reader *, int,
 					 const char *, size_t));
-static int glue_header_name	PARAMS ((cpp_reader *, cpp_token *));
-static int  parse_include	PARAMS ((cpp_reader *, cpp_token *));
+static const cpp_token *glue_header_name PARAMS ((cpp_reader *));
+static const cpp_token *parse_include PARAMS ((cpp_reader *));
 static void push_conditional	PARAMS ((cpp_reader *, int, int,
 					 const cpp_hashnode *));
 static unsigned int read_flag	PARAMS ((cpp_reader *, unsigned int));
@@ -100,7 +100,8 @@ static void do_pragma_once	PARAMS ((cpp_reader *));
 static void do_pragma_poison	PARAMS ((cpp_reader *));
 static void do_pragma_system_header	PARAMS ((cpp_reader *));
 static void do_pragma_dependency	PARAMS ((cpp_reader *));
-static int get__Pragma_string	PARAMS ((cpp_reader *, cpp_token *));
+static const cpp_token *get_token_no_padding PARAMS ((cpp_reader *));
+static const cpp_token *get__Pragma_string PARAMS ((cpp_reader *));
 static unsigned char *destringize	PARAMS ((const cpp_string *,
 						 unsigned int *));
 static int parse_answer PARAMS ((cpp_reader *, struct answer **, int));
@@ -485,13 +486,13 @@ do_undef (pfile)
 
 /* Helper routine used by parse_include.  Reinterpret the current line
    as an h-char-sequence (< ... >); we are looking at the first token
-   after the <.  Returns zero on success.  */
-static int
-glue_header_name (pfile, header)
+   after the <.  Returns the header as a token, or NULL on failure.  */
+static const cpp_token *
+glue_header_name (pfile)
      cpp_reader *pfile;
-     cpp_token *header;
 {
-  cpp_token token;
+  cpp_token *header = NULL;
+  const cpp_token *token;
   unsigned char *buffer, *token_mem;
   size_t len, total_len = 0, capacity = 1024;
 
@@ -501,25 +502,25 @@ glue_header_name (pfile, header)
   buffer = (unsigned char *) xmalloc (capacity);
   for (;;)
     {
-      cpp_get_token (pfile, &token);
+      token = cpp_get_token (pfile);
 
-      if (token.type == CPP_GREATER || token.type == CPP_EOF)
+      if (token->type == CPP_GREATER || token->type == CPP_EOF)
 	break;
 
-      len = cpp_token_len (&token);
+      len = cpp_token_len (token);
       if (total_len + len > capacity)
 	{
 	  capacity = (capacity + len) * 2;
 	  buffer = (unsigned char *) xrealloc (buffer, capacity);
 	}
 
-      if (token.flags & PREV_WHITE)
+      if (token->flags & PREV_WHITE)
 	buffer[total_len++] = ' ';
 
-      total_len = cpp_spell_token (pfile, &token, &buffer[total_len]) - buffer;
+      total_len = cpp_spell_token (pfile, token, &buffer[total_len]) - buffer;
     }
 
-  if (token.type == CPP_EOF)
+  if (token->type == CPP_EOF)
     cpp_error (pfile, "missing terminating > character");
   else
     {
@@ -527,6 +528,7 @@ glue_header_name (pfile, header)
       memcpy (token_mem, buffer, total_len);
       token_mem[total_len] = '\0';
 
+      header = _cpp_temp_token (pfile);
       header->type = CPP_HEADER_NAME;
       header->flags &= ~PREV_WHITE;
       header->val.str.len = total_len;
@@ -534,17 +536,17 @@ glue_header_name (pfile, header)
     }
 
   free ((PTR) buffer);
-  return token.type == CPP_EOF;
+  return header;
 }
 
-/* Parse the header name of #include, #include_next, #import and
-   #pragma dependency.  Returns zero on success.  */
-static int
-parse_include (pfile, header)
+/* Returns the header string of #include, #include_next, #import and
+   #pragma dependency.  Returns NULL on error.  */
+static const cpp_token *
+parse_include (pfile)
      cpp_reader *pfile;
-     cpp_token *header;
 {
   const unsigned char *dir;
+  const cpp_token *header;
 
   if (pfile->directive == &dtable[T_PRAGMA])
     dir = U"pragma dependency";
@@ -552,25 +554,27 @@ parse_include (pfile, header)
     dir = pfile->directive->name;
 
   /* Allow macro expansion.  */
-  cpp_get_token (pfile, header);
+  header = cpp_get_token (pfile);
   if (header->type != CPP_STRING && header->type != CPP_HEADER_NAME)
     {
       if (header->type != CPP_LESS)
 	{
 	  cpp_error (pfile, "#%s expects \"FILENAME\" or <FILENAME>", dir);
-	  return 1;
+	  return NULL;
 	}
-      if (glue_header_name (pfile, header))
-	return 1;
+
+      header = glue_header_name (pfile);
+      if (header == NULL)
+	return header;
     }
 
   if (header->val.str.len == 0)
     {
       cpp_error (pfile, "empty file name in #%s", dir);
-      return 1;
+      return NULL;
     }
 
-  return 0;
+  return header;
 }
 
 /* Handle #include, #include_next and #import.  */
@@ -579,7 +583,7 @@ do_include_common (pfile, type)
      cpp_reader *pfile;
      enum include_type type;
 {
-  cpp_token header;
+  const cpp_token *header;
 
   /* For #include_next, if this is the primary source file, warn and
      use the normal search logic.  */
@@ -595,7 +599,8 @@ do_include_common (pfile, type)
 	   "#import is obsolete, use an #ifndef wrapper in the header file");
     }
 
-  if (!parse_include (pfile, &header))
+  header = parse_include (pfile);
+  if (header)
     {
       /* Prevent #include recursion.  */
       if (pfile->line_maps.depth >= CPP_STACK_MAX)
@@ -607,9 +612,9 @@ do_include_common (pfile, type)
 	  skip_rest_of_line (pfile);
 	  if (pfile->cb.include)
 	    (*pfile->cb.include) (pfile, pfile->directive_line,
-				  pfile->directive->name, &header);
+				  pfile->directive->name, header);
 
-	  _cpp_execute_include (pfile, &header, type);
+	  _cpp_execute_include (pfile, header, type);
 	}
     }
 }
@@ -693,7 +698,7 @@ static void
 do_line (pfile)
      cpp_reader *pfile;
 {
-  cpp_token token;
+  const cpp_token *token;
   const char *new_file = pfile->map->to_file;
   unsigned long new_lineno;
   unsigned int cap, new_sysp = pfile->map->sysp;
@@ -708,12 +713,13 @@ do_line (pfile)
     _cpp_backup_tokens (pfile, 1);
 
   /* #line commands expand macros.  */
-  cpp_get_token (pfile, &token);
-  if (token.type != CPP_NUMBER
-      || strtoul_for_line (token.val.str.text, token.val.str.len, &new_lineno))
+  token = cpp_get_token (pfile);
+  if (token->type != CPP_NUMBER
+      || strtoul_for_line (token->val.str.text, token->val.str.len,
+			   &new_lineno))
     {
       cpp_error (pfile, "\"%s\" after #line is not a positive integer",
-		 cpp_token_as_text (pfile, &token));
+		 cpp_token_as_text (pfile, token));
       return;
     }      
 
@@ -721,10 +727,10 @@ do_line (pfile)
       && (new_lineno == 0 || new_lineno > cap))
     cpp_pedwarn (pfile, "line number out of range");
 
-  cpp_get_token (pfile, &token);
-  if (token.type == CPP_STRING)
+  token = cpp_get_token (pfile);
+  if (token->type == CPP_STRING)
     {
-      new_file = (const char *) token.val.str.text;
+      new_file = (const char *) token->val.str.text;
 
       /* Only accept flags for the # 55 form.  */
       if (pfile->state.line_extension)
@@ -755,10 +761,10 @@ do_line (pfile)
 	}
       check_eol (pfile);
     }
-  else if (token.type != CPP_EOF)
+  else if (token->type != CPP_EOF)
     {
       cpp_error (pfile, "\"%s\" is not a valid filename",
-		 cpp_token_as_text (pfile, &token));
+		 cpp_token_as_text (pfile, token));
       return;
     }
 
@@ -827,13 +833,12 @@ static void
 do_ident (pfile)
      cpp_reader *pfile;
 {
-  cpp_token str;
+  const cpp_token *str = cpp_get_token (pfile);
 
-  cpp_get_token (pfile, &str);
-  if (str.type != CPP_STRING)
-    cpp_error (pfile, "invalid #ident");
+  if (str->type != CPP_STRING)
+    cpp_error (pfile, "invalid #ident directive");
   else if (pfile->cb.ident)
-    (*pfile->cb.ident) (pfile, pfile->directive_line, &str.val.str);
+    (*pfile->cb.ident) (pfile, pfile->directive_line, &str->val.str);
 
   check_eol (pfile);
 }
@@ -950,7 +955,7 @@ do_pragma (pfile)
 {
   pragma_cb handler = NULL;
   const struct pragma_entry *p;
-  cpp_token tok;
+  const cpp_token *token;
   unsigned int count = 0;
 
   p = pfile->pragmas;
@@ -958,10 +963,10 @@ do_pragma (pfile)
 
  new_space:
   count++;
-  cpp_get_token (pfile, &tok);
-  if (tok.type == CPP_NAME)
+  token = cpp_get_token (pfile);
+  if (token->type == CPP_NAME)
     {
-      const cpp_hashnode *node = tok.val.node;
+      const cpp_hashnode *node = token->val.node;
       size_t len = NODE_LEN (node);
 
       while (p)
@@ -990,7 +995,7 @@ do_pragma (pfile)
      themselves.  Stand-alone CPP must ignore us, otherwise it will
      prefix the directive with spaces, hence the 1.  Ugh.  */
   if (pfile->cb.line_change)
-    (*pfile->cb.line_change)(pfile, &tok, 1);
+    (*pfile->cb.line_change)(pfile, token, 1);
 
   if (handler)
     (*handler) (pfile);
@@ -1078,22 +1083,22 @@ static void
 do_pragma_dependency (pfile)
      cpp_reader *pfile;
 {
-  cpp_token header, msg;
+  const cpp_token *header;
   int ordering;
  
-  if (parse_include (pfile, &header))
+  header = parse_include (pfile);
+  if (!header)
     return;
 
-  ordering = _cpp_compare_file_date (pfile, &header);
+  ordering = _cpp_compare_file_date (pfile, header);
   if (ordering < 0)
     cpp_warning (pfile, "cannot find source %s",
-		 cpp_token_as_text (pfile, &header));
+		 cpp_token_as_text (pfile, header));
   else if (ordering > 0)
     {
       cpp_warning (pfile, "current file is older than %s",
-		   cpp_token_as_text (pfile, &header));
-      cpp_get_token (pfile, &msg);
-      if (msg.type != CPP_EOF)
+		   cpp_token_as_text (pfile, header));
+      if (cpp_get_token (pfile)->type != CPP_EOF)
 	{
 	  _cpp_backup_tokens (pfile, 1);
 	  do_diagnostic (pfile, WARNING, 0);
@@ -1101,24 +1106,38 @@ do_pragma_dependency (pfile)
     }
 }
 
-/* Check syntax is "(string-literal)".  Returns 0 on success.  */
-static int
-get__Pragma_string (pfile, string)
+/* Get a token but skip padding.  */
+static const cpp_token *
+get_token_no_padding (pfile)
      cpp_reader *pfile;
-     cpp_token *string;
 {
-  cpp_token paren;
+  for (;;)
+    {
+      const cpp_token *result = cpp_get_token (pfile);
+      if (result->type != CPP_PADDING)
+	return result;
+    }
+}
 
-  cpp_get_token (pfile, &paren);
-  if (paren.type != CPP_OPEN_PAREN)
-    return 1;
+/* Check syntax is "(string-literal)".  Returns the string on success,
+   or NULL on failure.  */
+static const cpp_token *
+get__Pragma_string (pfile)
+     cpp_reader *pfile;
+{
+  const cpp_token *string;
 
-  cpp_get_token (pfile, string);
+  if (get_token_no_padding (pfile)->type != CPP_OPEN_PAREN)
+    return NULL;
+
+  string = get_token_no_padding (pfile);
   if (string->type != CPP_STRING && string->type != CPP_WSTRING)
-    return 1;
+    return NULL;
+
+  if (get_token_no_padding (pfile)->type != CPP_CLOSE_PAREN)
+    return NULL;
 
-  cpp_get_token (pfile, &paren);
-  return paren.type != CPP_CLOSE_PAREN;
+  return string;
 }
 
 /* Returns a malloced buffer containing a destringized cpp_string by
@@ -1148,11 +1167,11 @@ void
 _cpp_do__Pragma (pfile)
      cpp_reader *pfile;
 {
-  cpp_token string;
+  const cpp_token *string = get__Pragma_string (pfile);
   unsigned char *buffer;
   unsigned int len;
 
-  if (get__Pragma_string (pfile, &string))
+  if (!string)
     cpp_error (pfile, "_Pragma takes a parenthesized string literal");
   else
     {
@@ -1167,7 +1186,7 @@ _cpp_do__Pragma (pfile)
 	 Getting these correct line markers is a little tricky.  */
 
       unsigned int orig_line = pfile->line;
-      buffer = destringize (&string.val.str, &len);
+      buffer = destringize (&string->val.str, &len);
       run_directive (pfile, T_PRAGMA, (char *) buffer, len);
       free ((PTR) buffer);
       pfile->line = orig_line;
@@ -1386,7 +1405,7 @@ parse_answer (pfile, answerp, type)
      struct answer **answerp;
      int type;
 {
-  cpp_token paren, *token;
+  const cpp_token *paren;
   struct answer *answer;
 
   if (POOL_FRONT (&pfile->macro_pool) + sizeof (struct answer) >
@@ -1397,10 +1416,10 @@ parse_answer (pfile, answerp, type)
 
   /* In a conditional, it is legal to not have an open paren.  We
      should save the following token in this case.  */
-  cpp_get_token (pfile, &paren);
+  paren = cpp_get_token (pfile);
 
   /* If not a paren, see if we're OK.  */
-  if (paren.type != CPP_OPEN_PAREN)
+  if (paren->type != CPP_OPEN_PAREN)
     {
       /* In a conditional no answer is a test for any answer.  It
          could be followed by any token.  */
@@ -1411,7 +1430,7 @@ parse_answer (pfile, answerp, type)
 	}
 
       /* #unassert with no answer is valid - it removes all answers.  */
-      if (type == T_UNASSERT && paren.type == CPP_EOF)
+      if (type == T_UNASSERT && paren->type == CPP_EOF)
 	return 0;
 
       cpp_error (pfile, "missing '(' after predicate");
@@ -1420,7 +1439,7 @@ parse_answer (pfile, answerp, type)
 
   for (;;)
     {
-      token = &answer->first[answer->count];
+      cpp_token *token = &answer->first[answer->count];
       /* Check we have room for the token.  */
       if ((unsigned char *) (token + 1) >= POOL_LIMIT (&pfile->macro_pool))
 	{
@@ -1429,7 +1448,7 @@ parse_answer (pfile, answerp, type)
 	  token = &answer->first[answer->count];
 	}
 
-      cpp_get_token (pfile, token);
+      *token = *cpp_get_token (pfile);
       if (token->type == CPP_CLOSE_PAREN)
 	break;
 
@@ -1466,25 +1485,25 @@ parse_assertion (pfile, answerp, type)
      int type;
 {
   cpp_hashnode *result = 0;
-  cpp_token predicate;
+  const cpp_token *predicate;
 
   /* We don't expand predicates or answers.  */
   pfile->state.prevent_expansion++;
 
   *answerp = 0;
-  cpp_get_token (pfile, &predicate);
-  if (predicate.type == CPP_EOF)
+  predicate = cpp_get_token (pfile);
+  if (predicate->type == CPP_EOF)
     cpp_error (pfile, "assertion without predicate");
-  else if (predicate.type != CPP_NAME)
+  else if (predicate->type != CPP_NAME)
     cpp_error (pfile, "predicate must be an identifier");
   else if (parse_answer (pfile, answerp, type) == 0)
     {
-      unsigned int len = NODE_LEN (predicate.val.node);
+      unsigned int len = NODE_LEN (predicate->val.node);
       unsigned char *sym = alloca (len + 1);
 
       /* Prefix '#' to get it out of macro namespace.  */
       sym[0] = '#';
-      memcpy (sym + 1, NODE_NAME (predicate.val.node), len);
+      memcpy (sym + 1, NODE_NAME (predicate->val.node), len);
       result = cpp_lookup (pfile, sym, len + 1);
     }
 
diff --git a/gcc/cpplib.h b/gcc/cpplib.h
index ad7418755a5..b8db3663794 100644
--- a/gcc/cpplib.h
+++ b/gcc/cpplib.h
@@ -134,6 +134,7 @@ struct file_name_map_list;
 \
   TK(CPP_COMMENT,	SPELL_STRING)	/* Only if output comments.  */ \
   TK(CPP_MACRO_ARG,	SPELL_NONE)	/* Macro argument.  */		\
+  OP(CPP_PADDING,	"")		/* Whitespace for cpp0.  */	\
   OP(CPP_EOF,		"EOL")		/* End of line or file.  */
 
 #define OP(e, s) e,
@@ -164,8 +165,7 @@ struct cpp_string
 #define PASTE_LEFT	(1 << 3) /* If on LHS of a ## operator.  */
 #define NAMED_OP	(1 << 4) /* C++ named operators.  */
 #define NO_EXPAND	(1 << 5) /* Do not macro-expand this token.  */
-#define AVOID_LPASTE	(1 << 6) /* Check left for accidental pastes.  */
-#define BOL		(1 << 7) /* Token at beginning of line.  */
+#define BOL		(1 << 6) /* Token at beginning of line.  */
 
 /* A preprocessing token.  This has been carefully packed and should
    occupy 12 bytes on 32-bit hosts and 16 bytes on 64-bit hosts.  */
@@ -179,6 +179,7 @@ struct cpp_token
   union
   {
     cpp_hashnode *node;		/* An identifier.  */
+    const cpp_token *source;	/* Inherit padding from this token.  */
     struct cpp_string str;	/* A string, or number.  */
     unsigned int arg_no;	/* Argument no. for a CPP_MACRO_ARG.  */
     unsigned char c;		/* Character represented by CPP_OTHER.  */
@@ -235,6 +236,9 @@ struct cpp_options
   /* The language we're preprocessing.  */
   enum c_lang lang;
 
+  /* Nonzero means to return spacing characters for stand-alone CPP.  */
+  unsigned char spacing;
+
   /* Non-0 means -v, so print the full set of include dirs.  */
   unsigned char verbose;
 
@@ -497,7 +501,7 @@ extern int cpp_avoid_paste PARAMS ((cpp_reader *, const cpp_token *,
 				    const cpp_token *));
 extern enum cpp_ttype cpp_can_paste PARAMS ((cpp_reader *, const cpp_token *,
 					     const cpp_token *, int *));
-extern void cpp_get_token PARAMS ((cpp_reader *, cpp_token *));
+extern const cpp_token *cpp_get_token PARAMS ((cpp_reader *));
 extern const unsigned char *cpp_macro_definition PARAMS ((cpp_reader *,
 						  const cpp_hashnode *));
 extern void _cpp_backup_tokens PARAMS ((cpp_reader *, unsigned int));
diff --git a/gcc/cppmacro.c b/gcc/cppmacro.c
index e5260694c42..6cc45022288 100644
--- a/gcc/cppmacro.c
+++ b/gcc/cppmacro.c
@@ -45,9 +45,9 @@ struct cpp_macro
 typedef struct macro_arg macro_arg;
 struct macro_arg
 {
-  cpp_token *first;		/* First token in unexpanded argument.  */
-  cpp_token *expanded;		/* Macro-expanded argument.   */
-  cpp_token *stringified;	/* Stringified argument.  */
+  const cpp_token **first;	/* First token in unexpanded argument.  */
+  const cpp_token **expanded;	/* Macro-expanded argument.   */
+  const cpp_token *stringified;	/* Stringified argument.  */
   unsigned int count;		/* # of tokens in argument.  */
   unsigned int expanded_count;	/* # of tokens in expanded argument.  */
 };
@@ -57,25 +57,29 @@ struct macro_arg
 static void lock_pools PARAMS ((cpp_reader *));
 static void unlock_pools PARAMS ((cpp_reader *));
 static int enter_macro_context PARAMS ((cpp_reader *, cpp_hashnode *));
-static void builtin_macro PARAMS ((cpp_reader *, cpp_token *));
-static cpp_context *push_arg_context PARAMS ((cpp_reader *, macro_arg *));
+static const cpp_token *builtin_macro PARAMS ((cpp_reader *, cpp_hashnode *));
+static void push_token_context
+  PARAMS ((cpp_reader *, cpp_macro *, const cpp_token *, unsigned int));
+static void push_ptoken_context
+  PARAMS ((cpp_reader *, cpp_macro *, const cpp_token **, unsigned int));
 static enum cpp_ttype parse_arg PARAMS ((cpp_reader *, macro_arg *, int));
 static macro_arg *parse_args PARAMS ((cpp_reader *, const cpp_hashnode *));
 static cpp_context *next_context PARAMS ((cpp_reader *));
+static const cpp_token *padding_token
+  PARAMS ((cpp_reader *, const cpp_token *));
 static void expand_arg PARAMS ((cpp_reader *, macro_arg *));
 static unsigned char *quote_string PARAMS ((unsigned char *,
 					    const unsigned char *,
 					    unsigned int));
-static void make_string_token PARAMS ((cpp_pool *, cpp_token *,
-				       const U_CHAR *, unsigned int));
-static void make_number_token PARAMS ((cpp_reader *, cpp_token *, int));
-static void stringify_arg PARAMS ((cpp_reader *, macro_arg *));
-static void paste_all_tokens PARAMS ((cpp_reader *, cpp_token *));
-static int paste_tokens PARAMS ((cpp_reader *, cpp_token *, cpp_token *));
-static int funlike_invocation_p PARAMS ((cpp_reader *, const cpp_hashnode *,
-					  struct toklist *));
-static void replace_args PARAMS ((cpp_reader *, cpp_macro *, macro_arg *,
-				  struct toklist *));
+static const cpp_token *new_string_token PARAMS ((cpp_reader *, U_CHAR *,
+						  unsigned int));
+static const cpp_token *new_number_token PARAMS ((cpp_reader *, int));
+static const cpp_token *stringify_arg PARAMS ((cpp_reader *, macro_arg *));
+static void paste_all_tokens PARAMS ((cpp_reader *, const cpp_token *));
+static int paste_tokens PARAMS ((cpp_reader *, cpp_token *,
+				 const cpp_token *));
+static int funlike_invocation_p PARAMS ((cpp_reader *, const cpp_hashnode *));
+static void replace_args PARAMS ((cpp_reader *, cpp_macro *, macro_arg *));
 
 /* #define directive parsing and handling.  */
 
@@ -89,32 +93,31 @@ static void check_trad_stringification PARAMS ((cpp_reader *,
 						const cpp_macro *,
 						const cpp_string *));
 
-/* Allocates a buffer to hold a token's TEXT, and converts TOKEN to a
-   CPP_STRING token containing TEXT in quoted form.  */
-static void
-make_string_token (pool, token, text, len)
-     cpp_pool *pool;
-     cpp_token *token;
-     const U_CHAR *text;
+/* Allocates and returns a CPP_STRING token, containing TEXT of length
+   LEN, after null-terminating it.  TEXT must be in permanent storage.  */
+static const cpp_token *
+new_string_token (pfile, text, len)
+     cpp_reader *pfile;
+     unsigned char *text;
      unsigned int len;
 {
-  U_CHAR *buf = _cpp_pool_alloc (pool, len * 4 + 1);
+  cpp_token *token = _cpp_temp_token (pfile);
 
+  text[len] = '\0';
   token->type = CPP_STRING;
-  token->val.str.text = buf;
-  token->val.str.len = quote_string (buf, text, len) - buf;
-  buf[token->val.str.len] = '\0';
+  token->val.str.len = len;
+  token->val.str.text = text;
   token->flags = 0;
+  return token;
 }
 
-/* Allocates and converts a temporary token to a CPP_NUMBER token,
-   evaluating to NUMBER.  */
-static void
-make_number_token (pfile, token, number)
+/* Allocates and returns a CPP_NUMBER token evaluating to NUMBER.  */
+static const cpp_token *
+new_number_token (pfile, number)
      cpp_reader *pfile;
-     cpp_token *token;
      int number;
 {
+  cpp_token *token = _cpp_temp_token (pfile);
   unsigned char *buf = _cpp_pool_alloc (&pfile->ident_pool, 20);
 
   sprintf ((char *) buf, "%d", number);
@@ -122,6 +125,7 @@ make_number_token (pfile, token, number)
   token->val.str.text = buf;
   token->val.str.len = ustrlen (buf);
   token->flags = 0;
+  return token;
 }
 
 static const char * const monthnames[] =
@@ -131,20 +135,23 @@ static const char * const monthnames[] =
 };
 
 /* Handle builtin macros like __FILE__.  */
-static void
-builtin_macro (pfile, token)
+static const cpp_token *
+builtin_macro (pfile, node)
      cpp_reader *pfile;
-     cpp_token *token;
+     cpp_hashnode *node;
 {
-  unsigned char flags = ((token->flags & (PREV_WHITE | BOL)) | AVOID_LPASTE);
-  cpp_hashnode *node = token->val.node;
-
   switch (node->value.builtin)
     {
+    default:
+      cpp_ice (pfile, "invalid builtin macro \"%s\"", NODE_NAME (node));
+      return new_number_token (pfile, 1);
+
     case BT_FILE:
     case BT_BASE_FILE:
       {
+	unsigned int len;
 	const char *name;
+	U_CHAR *buf;
 	const struct line_map *map = pfile->map;
 
 	if (node->value.builtin == BT_BASE_FILE)
@@ -152,64 +159,63 @@ builtin_macro (pfile, token)
 	    map = INCLUDED_FROM (&pfile->line_maps, map);
 
 	name = map->to_file;
-	make_string_token (&pfile->ident_pool, token,
-			   (const unsigned char *) name, strlen (name));
+	len = strlen (name);
+	buf = _cpp_pool_alloc (&pfile->ident_pool, len * 4 + 1);
+	len = quote_string (buf, (const unsigned char *) name, len) - buf;
+
+	return new_string_token (pfile, buf, len);
       }
-      break;
 	
     case BT_INCLUDE_LEVEL:
       /* The line map depth counts the primary source as level 1, but
 	 historically __INCLUDE_DEPTH__ has called the primary source
 	 level 0.  */
-      make_number_token (pfile, token, pfile->line_maps.depth - 1);
-      break;
+      return new_number_token (pfile, pfile->line_maps.depth - 1);
 
     case BT_SPECLINE:
       /* If __LINE__ is embedded in a macro, it must expand to the
 	 line of the macro's invocation, not its definition.
 	 Otherwise things like assert() will not work properly.  */
-      make_number_token (pfile, token,
-			 SOURCE_LINE (pfile->map, pfile->cur_token[-1].line));
-      break;
+      return new_number_token (pfile, SOURCE_LINE (pfile->map,
+						   pfile->cur_token[-1].line));
 
     case BT_STDC:
       {
 	int stdc = (!CPP_IN_SYSTEM_HEADER (pfile)
 		    || pfile->spec_nodes.n__STRICT_ANSI__->type != NT_VOID);
-	make_number_token (pfile, token, stdc);
+	return new_number_token (pfile, stdc);
       }
-      break;
 
     case BT_DATE:
     case BT_TIME:
       if (pfile->date.type == CPP_EOF)
 	{
-	  /* Allocate __DATE__ and __TIME__ from permanent storage,
-	     and save them in pfile so we don't have to do this again.
-	     We don't generate these strings at init time because
-	     time() and localtime() are very slow on some systems.  */
+	  /* Allocate __DATE__ and __TIME__ strings from permanent
+	     storage.  We only do this once, and don't generate them
+	     at init time, because time() and localtime() are very
+	     slow on some systems.  */
 	  time_t tt = time (NULL);
 	  struct tm *tb = localtime (&tt);
 
-	  make_string_token (&pfile->ident_pool, &pfile->date,
-			     DSC("Oct 11 1347"));
-	  make_string_token (&pfile->ident_pool, &pfile->time,
-			     DSC("12:34:56"));
-
+	  pfile->date.val.str.text =
+	    _cpp_pool_alloc (&pfile->ident_pool, sizeof ("Oct 11 1347"));
+	  pfile->date.val.str.len = sizeof ("Oct 11 1347") - 1;
+	  pfile->date.type = CPP_STRING;
+	  pfile->date.flags = 0;
 	  sprintf ((char *) pfile->date.val.str.text, "%s %2d %4d",
 		   monthnames[tb->tm_mon], tb->tm_mday, tb->tm_year + 1900);
+
+	  pfile->time.val.str.text =
+	    _cpp_pool_alloc (&pfile->ident_pool, sizeof ("12:34:56"));
+	  pfile->time.val.str.len = sizeof ("12:34:56") - 1;
+	  pfile->time.type = CPP_STRING;
+	  pfile->time.flags = 0;
 	  sprintf ((char *) pfile->time.val.str.text, "%02d:%02d:%02d",
 		   tb->tm_hour, tb->tm_min, tb->tm_sec);
 	}
-      *token = node->value.builtin == BT_DATE ? pfile->date: pfile->time;
-      break;
 
-    default:
-      cpp_ice (pfile, "invalid builtin macro \"%s\"", NODE_NAME (node));
-      break;
+      return node->value.builtin == BT_DATE ? &pfile->date: &pfile->time;
     }
-
-  token->flags = flags;
 }
 
 static void
@@ -260,7 +266,7 @@ quote_string (dest, src, len)
 
 /* Convert a token sequence to a single string token according to the
    rules of the ISO C #-operator.  */
-static void
+static const cpp_token *
 stringify_arg (pfile, arg)
      cpp_reader *pfile;
      macro_arg *arg;
@@ -268,17 +274,26 @@ stringify_arg (pfile, arg)
   cpp_pool *pool = &pfile->ident_pool;
   unsigned char *start = POOL_FRONT (pool);
   unsigned int i, escape_it, total_len = 0, backslash_count = 0;
+  const cpp_token *source = NULL;
 
   /* Loop, reading in the argument's tokens.  */
   for (i = 0; i < arg->count; i++)
     {
       unsigned char *dest;
-      const cpp_token *token = &arg->first[i];
-      unsigned int len = cpp_token_len (token);
+      const cpp_token *token = arg->first[i];
+      unsigned int len;
+
+      if (token->type == CPP_PADDING)
+	{
+	  if (source == NULL)
+	    source = token->val.source;
+	  continue;
+	}
 
       escape_it = (token->type == CPP_STRING || token->type == CPP_WSTRING
 		   || token->type == CPP_CHAR || token->type == CPP_WCHAR);
 
+      len = cpp_token_len (token);
       if (escape_it)
 	/* Worst case is each char is octal.  */
 	len *= 4;
@@ -291,9 +306,15 @@ stringify_arg (pfile, arg)
 	  dest = &start[total_len];
 	}
 
-      /* No leading white space.  */
-      if (token->flags & PREV_WHITE && total_len > 0)
-	*dest++ = ' ';
+      /* Leading white space?  */
+      if (total_len)
+	{
+	  if (source == NULL)
+	    source = token;
+	  if (source->flags & PREV_WHITE)
+	    *dest++ = ' ';
+	}
+      source = NULL;
 
       if (escape_it)
 	{
@@ -320,15 +341,9 @@ stringify_arg (pfile, arg)
       total_len--;
     }
 
-  /* Null terminate, and commit the memory.  */
-  start[total_len] = '\0';
+  /* Commit the memory, including NUL, and return the token.  */
   POOL_COMMIT (pool, total_len + 1);
-
-  arg->stringified = xnew (cpp_token);
-  arg->stringified->flags = 0;
-  arg->stringified->type = CPP_STRING;
-  arg->stringified->val.str.text = start;
-  arg->stringified->val.str.len = total_len;
+  return new_string_token (pfile, start, total_len);
 }
 
 /* Try to paste two tokens.  On success, the LHS becomes the pasted
@@ -337,9 +352,10 @@ stringify_arg (pfile, arg)
 static int
 paste_tokens (pfile, lhs, rhs)
      cpp_reader *pfile;
-     cpp_token *lhs, *rhs;
+     cpp_token *lhs;
+     const cpp_token *rhs;
 {
-  unsigned char flags;
+  unsigned char flags = 0;
   int digraph = 0;
   enum cpp_ttype type;
 
@@ -353,20 +369,9 @@ paste_tokens (pfile, lhs, rhs)
 	 "pasting \"%s\" and \"%s\" does not give a valid preprocessing token",
 		     cpp_token_as_text (pfile, lhs),
 		     cpp_token_as_text (pfile, rhs));
-
-      /* The standard states that behaviour is undefined.  By the
-         principle of least surpise, we step back before the RHS, and
-         mark it to prevent macro expansion.  Tests in the testsuite
-         rely on clearing PREV_WHITE here, though you could argue we
-         should actually set it.  Assembler can have '.' in labels and
-         so requires that we don't insert spaces there.  Maybe we should
-	 change this to put out a space unless it's assembler.  */
-      rhs->flags &= ~PREV_WHITE;
-      rhs->flags |= NO_EXPAND;
       return 1;
     }
 
-  flags = lhs->flags & ~DIGRAPH;
   if (digraph)
     flags |= DIGRAPH;
 
@@ -416,10 +421,17 @@ paste_tokens (pfile, lhs, rhs)
 static void
 paste_all_tokens (pfile, lhs)
      cpp_reader *pfile;
-     cpp_token *lhs;
+     const cpp_token *lhs;
 {
-  cpp_token *rhs;
-  unsigned char orig_flags = lhs->flags;
+  cpp_token *pasted;
+  const cpp_token *rhs;
+  cpp_context *context = pfile->context;
+
+  /* Copy lhs to pasted, but preserve original line and column.  */
+  pasted = _cpp_temp_token (pfile);
+  pasted->type = lhs->type;
+  pasted->flags = lhs->flags;
+  pasted->val.str = lhs->val.str;
 
   do
     {
@@ -428,20 +440,25 @@ paste_all_tokens (pfile, lhs)
 	 object-like macro, or a function-like macro with arguments
 	 inserted.  In either case, the constraints to #define
 	 guarantee we have at least one more token.  */
-      rhs = pfile->context->list.first++;
-      if (paste_tokens (pfile, lhs, rhs))
+      if (context->direct_p)
+	rhs = context->first.token++;
+      else
+	rhs = *context->first.ptoken++;
+
+      if (rhs->type == CPP_PADDING)
+	abort ();
+
+      if (paste_tokens (pfile, pasted, rhs))
 	{
-	  /* We failed.  Step back so we read the RHS in next.  */
-	  pfile->context->list.first--;
+	  _cpp_backup_tokens (pfile, 1);
 	  break;
 	}
     }
   while (rhs->flags & PASTE_LEFT);
 
-  /* The pasted token has the PREV_WHITE flag of the LHS, is no longer
-     PASTE_LEFT, and is subject to macro expansion.  */
-  lhs->flags &= ~(PREV_WHITE | BOL | PASTE_LEFT | NO_EXPAND);
-  lhs->flags |= orig_flags & (PREV_WHITE | BOL | AVOID_LPASTE);
+  /* Clear PASTE_LEFT flag, put the token in its own context.  */
+  pasted->flags &= ~PASTE_LEFT;
+  push_token_context (pfile, NULL, pasted, 1);
 }
 
 /* Reads the unexpanded tokens of a macro argument into ARG.  VAR_ARGS
@@ -455,26 +472,24 @@ parse_arg (pfile, arg, variadic)
 {
   enum cpp_ttype result;
   unsigned int paren = 0;
-  unsigned int line;
 
-  arg->first = (cpp_token *) POOL_FRONT (&pfile->argument_pool);
+  arg->first = (const cpp_token **) POOL_FRONT (&pfile->argument_pool);
   for (;; arg->count++)
     {
-      cpp_token *token = &arg->first[arg->count];
-      if ((unsigned char *) (token + 1) >= POOL_LIMIT (&pfile->argument_pool))
+      const cpp_token *token;
+      const cpp_token **ptoken = &arg->first[arg->count];
+      if ((unsigned char *) (ptoken + 2) >= POOL_LIMIT (&pfile->argument_pool))
 	{
-	  _cpp_next_chunk (&pfile->argument_pool, sizeof (cpp_token),
+	  _cpp_next_chunk (&pfile->argument_pool, 2 * sizeof (cpp_token *),
 			   (unsigned char **) &arg->first);
-	  token = &arg->first[arg->count];
+	  ptoken = &arg->first[arg->count];
 	}
 
-      /* Newlines in arguments are white space (6.10.3.10).  */
-      line = pfile->line;
-      cpp_get_token (pfile, token);
-
-      if (line != pfile->line)
-	token->flags |= PREV_WHITE;
-
+      /* Drop leading padding.  */
+      do
+	token = cpp_get_token (pfile);
+      while (arg->count == 0 && token->type == CPP_PADDING);
+      *ptoken++ = token;
       result = token->type;
 
       if (result == CPP_OPEN_PAREN)
@@ -511,12 +526,15 @@ parse_arg (pfile, arg, variadic)
 	}
     }
 
+  /* Drop trailing padding.  */
+  while (arg->count > 0 && arg->first[arg->count - 1]->type == CPP_PADDING)
+    arg->count--;
+
   /* Commit the memory used to store the arguments.  We make the last
      argument a CPP_EOF, so that it terminates macro pre-expansion,
      but it is not included in arg->count.  */
-  arg->first[arg->count].type = CPP_EOF;  
-  POOL_COMMIT (&pfile->argument_pool, (arg->count + 1) * sizeof (cpp_token));
-
+  arg->first[arg->count] = &pfile->eof;  
+  POOL_COMMIT (&pfile->argument_pool, (arg->count + 1) * sizeof (cpp_token *));
   return result;
 }
 
@@ -599,22 +617,23 @@ parse_args (pfile, node)
 }
 
 static int
-funlike_invocation_p (pfile, node, list)
+funlike_invocation_p (pfile, node)
      cpp_reader *pfile;
      const cpp_hashnode *node;
-     struct toklist *list;
 {
-  cpp_token maybe_paren;
+  const cpp_token *maybe_paren;
   macro_arg *args = 0;
 
-  pfile->state.parsing_args = 1;
   pfile->state.prevent_expansion++;
-
   pfile->keep_tokens++;
-  cpp_get_token (pfile, &maybe_paren);
+
+  pfile->state.parsing_args = 1;
+  do
+    maybe_paren = cpp_get_token (pfile);
+  while (maybe_paren->type == CPP_PADDING);
   pfile->state.parsing_args = 2;
 
-  if (maybe_paren.type == CPP_OPEN_PAREN)
+  if (maybe_paren->type == CPP_OPEN_PAREN)
     args = parse_args (pfile, node);
   else
     {
@@ -625,14 +644,14 @@ funlike_invocation_p (pfile, node, list)
 		     NODE_NAME (node));
     }
 
-  pfile->state.prevent_expansion--;
   pfile->state.parsing_args = 0;
   pfile->keep_tokens--;
+  pfile->state.prevent_expansion--;
 
   if (args)
     {
       if (node->value.macro->paramc > 0)
-	replace_args (pfile, node->value.macro, args, list);
+	replace_args (pfile, node->value.macro, args);
       free (args);
     }
 
@@ -648,80 +667,60 @@ enter_macro_context (pfile, node)
      cpp_reader *pfile;
      cpp_hashnode *node;
 {
-  cpp_context *context;
-  cpp_macro *macro = node->value.macro;
-  struct toklist list;
-
-  /* Save the position of the outermost macro invocation.  */
-  if (!pfile->context->prev)
-    lock_pools (pfile);
-
-  if (macro->fun_like && !funlike_invocation_p (pfile, node, &list))
-    {
-      if (!pfile->context->prev)
-	unlock_pools (pfile);
-      return 0;
-    }
-
-  if (macro->paramc == 0)
+  if (node->flags & NODE_BUILTIN)
+    push_token_context (pfile, NULL, builtin_macro (pfile, node), 1);
+  else
     {
-      list.first = macro->expansion;
-      list.limit = macro->expansion + macro->count;
-    }
+      cpp_macro *macro = node->value.macro;
 
-  context = next_context (pfile);
-  context->list = list;
-  context->macro = macro;
-      
-  /* Disable the macro within its expansion.  */
-  macro->disabled = 1;
+      if (!pfile->context->prev)
+	lock_pools (pfile);
 
-  return 1;
-}
+      if (macro->fun_like && !funlike_invocation_p (pfile, node))
+	{
+	  if (!pfile->context->prev)
+	    unlock_pools (pfile);
+	  return 0;
+	}
 
-/* Move to the next context.  Create one if there is none.  */
-static cpp_context *
-next_context (pfile)
-     cpp_reader *pfile;
-{
-  cpp_context *prev = pfile->context;
-  cpp_context *result = prev->next;
+      /* Disable the macro within its expansion.  */
+      macro->disabled = 1;
 
-  if (result == 0)
-    {
-      result = xnew (cpp_context);
-      prev->next = result;
-      result->prev = prev;
-      result->next = 0;
+      if (macro->paramc == 0)
+	push_token_context (pfile, macro, macro->expansion, macro->count);
     }
-
-  pfile->context = result;
-  return result;
+ 
+  return 1;
 }
 
+/* Take the expansion of a function-like MACRO, replacing parameters
+   with the actual arguments.  Each instance is first macro-expanded,
+   unless that paramter is operated upon by the # or ## operators.  */
 static void
-replace_args (pfile, macro, args, list)
+replace_args (pfile, macro, args)
      cpp_reader *pfile;
      cpp_macro *macro;
      macro_arg *args;
-     struct toklist *list;
 {
-  unsigned char flags = 0;
   unsigned int i, total;
   const cpp_token *src, *limit;
-  cpp_token *dest;
+  const cpp_token **dest, **first;
   macro_arg *arg;
 
-  src = macro->expansion;
-  limit = src + macro->count;
-
   /* First, fully macro-expand arguments, calculating the number of
      tokens in the final expansion as we go.  This ensures that the
-     possible recursive use of argument_pool is fine.  */
-  total = limit - src;
-  for (; src < limit; src++)
+     possible recursive use of argument_pool is fine.  The ordering of
+     the if statements below is subtle; we must handle stringification
+     before pasting.  */
+  total = macro->count;
+  limit = macro->expansion + macro->count;
+
+  for (src = macro->expansion; src < limit; src++)
     if (src->type == CPP_MACRO_ARG)
       {
+	/* Leading and trailing padding tokens.  */
+	total += 2;
+
 	/* We have an argument.  If it is not being stringified or
 	   pasted it is macro-replaced before insertion.  */
 	arg = &args[src->val.arg_no - 1];
@@ -729,7 +728,7 @@ replace_args (pfile, macro, args, list)
 	if (src->flags & STRINGIFY_ARG)
 	  {
 	    if (!arg->stringified)
-	      stringify_arg (pfile, arg);
+	      arg->stringified = stringify_arg (pfile, arg);
 	  }
 	else if ((src->flags & PASTE_LEFT)
 		 || (src > macro->expansion && (src[-1].flags & PASTE_LEFT)))
@@ -737,113 +736,165 @@ replace_args (pfile, macro, args, list)
 	else
 	  {
 	    if (!arg->expanded)
-	      {
-		arg->expanded_count = 0;
-		if (arg->count)
-		  expand_arg (pfile, arg);
-	      }
+	      expand_arg (pfile, arg);
 	    total += arg->expanded_count - 1;
 	  }
       }
 
-  dest = (cpp_token *) _cpp_pool_alloc (&pfile->argument_pool,
-					total * sizeof (cpp_token));
-  list->first = dest;
+  /* Now allocate space for the expansion, copy the tokens and replace
+     the arguments.  */
+  first = (const cpp_token **) _cpp_pool_alloc (&pfile->argument_pool,
+						total * sizeof (cpp_token *));
+  dest = first;
 
   for (src = macro->expansion; src < limit; src++)
-    if (src->type == CPP_MACRO_ARG)
-      {
-	unsigned int count;
-	const cpp_token *from;
+    {
+      unsigned int count;
+      const cpp_token **from, **paste_flag;
 
-	arg = &args[src->val.arg_no - 1];
-	if (src->flags & STRINGIFY_ARG)
-	  {
-	    from = arg->stringified, count = 1;
-	    /* Ugh.  Maintain position of original argument.  */
-	    arg->stringified->line = src->line;
-	    arg->stringified->col = src->col;
-	  }
-	else if (src->flags & PASTE_LEFT)
-	  count = arg->count, from = arg->first;
-	else if (src > macro->expansion && (src[-1].flags & PASTE_LEFT))
-	  {
-	    count = arg->count, from = arg->first;
-	    if (dest != list->first)
-	      {
-		/* GCC has special semantics for , ## b where b is a
-		   varargs parameter: the comma disappears if b was
-		   given no actual arguments (not merely if b is an
-		   empty argument); otherwise pasting is turned off.  */
-		if (dest[-1].type == CPP_COMMA
-		    && macro->variadic
-		    && src->val.arg_no == macro->paramc)
-		  {
-		    if (count == 0)
-		      dest--;
-		    else
-		      dest[-1].flags &= ~PASTE_LEFT;
-		  }
-		/* Count == 0 is the RHS a placemarker case.  */
-		else if (count == 0)
-		  dest[-1].flags &= ~PASTE_LEFT;
-	      }
-	  }
-	else
-	  count = arg->expanded_count, from = arg->expanded;
+      if (src->type != CPP_MACRO_ARG)
+	{
+	  *dest++ = src;
+	  continue;
+	}
 
-	/* Count == 0 is the LHS a placemarker case.  */
-	if (count)
-	  {
-	    memcpy (dest, from, count * sizeof (cpp_token));
+      paste_flag = 0;
+      arg = &args[src->val.arg_no - 1];
+      if (src->flags & STRINGIFY_ARG)
+	count = 1, from = &arg->stringified;
+      else if (src->flags & PASTE_LEFT)
+	count = arg->count, from = arg->first;
+      else if (src != macro->expansion && (src[-1].flags & PASTE_LEFT))
+	{
+	  count = arg->count, from = arg->first;
+	  if (dest != first)
+	    {
+	      /* GCC has special semantics for , ## b where b is a
+		 varargs parameter: the comma disappears if b was
+		 given no actual arguments (not merely if b is an
+		 empty argument); otherwise the paste flag is removed.  */
+	      if (dest[-1]->type == CPP_COMMA
+		  && macro->variadic
+		  && src->val.arg_no == macro->paramc)
+		{
+		  if (count == 0)
+		    dest--;
+		  else
+		    paste_flag = dest - 1;
+		}
+	      /* Remove the paste flag if the RHS is a placemarker.  */
+	      else if (count == 0)
+		paste_flag = dest - 1;
+	    }
+	}
+      else
+	count = arg->expanded_count, from = arg->expanded;
 
-	    /* The first token gets PREV_WHITE of the CPP_MACRO_ARG.  */
-	    dest->flags &= ~(PREV_WHITE | BOL);
-	    dest->flags |= src->flags & (PREV_WHITE | BOL);
-	    dest->flags |= AVOID_LPASTE;
+      /* Padding on the left of an argument (unless RHS of ##).  */
+      if (!pfile->state.in_directive
+	  && src != macro->expansion && !(src[-1].flags & PASTE_LEFT))
+	*dest++ = padding_token (pfile, src);
 
-	    /* The last token gets the PASTE_LEFT of the CPP_MACRO_ARG.  */
-	    dest[count - 1].flags |= src->flags & PASTE_LEFT;
+      if (count)
+	{
+	  memcpy (dest, from, count * sizeof (cpp_token *));
+	  dest += count;
 
-	    dest += count;
-	  }
+	  /* With a non-empty argument on the LHS of ##, the last
+	     token should be flagged PASTE_LEFT.  */
+	  if (src->flags & PASTE_LEFT)
+	    paste_flag = dest - 1;
+	}
 
-	/* The token after the argument must avoid an accidental paste.  */
-	flags = AVOID_LPASTE;
-      }
-    else
-      {
-	*dest = *src;
-	dest->flags |= flags;
-	dest++;
-	flags = 0;
-      }
+      /* Avoid paste on RHS (even case count == 0).  */
+      if (!pfile->state.in_directive && !(src->flags & PASTE_LEFT))
+	*dest++ = &pfile->avoid_paste;
 
-  list->limit = dest;
+      /* Add a new paste flag, or remove an unwanted one.  */
+      if (paste_flag)
+	{
+	  cpp_token *token = _cpp_temp_token (pfile);
+	  token->type = (*paste_flag)->type;
+	  token->val.str = (*paste_flag)->val.str;
+	  if (src->flags & PASTE_LEFT)
+	    token->flags = (*paste_flag)->flags | PASTE_LEFT;
+	  else
+	    token->flags = (*paste_flag)->flags & ~PASTE_LEFT;
+	  *paste_flag = token;
+	}
+    }
 
   /* Free the expanded arguments.  */
   for (i = 0; i < macro->paramc; i++)
+    if (args[i].expanded)
+      free (args[i].expanded);
+
+  push_ptoken_context (pfile, macro, first, dest - first);
+}
+
+/* Return a special padding token, with padding inherited from SOURCE.  */
+static const cpp_token *
+padding_token (pfile, source)
+     cpp_reader *pfile;
+     const cpp_token *source;
+{
+  cpp_token *result = _cpp_temp_token (pfile);
+
+  result->type = CPP_PADDING;
+  result->val.source = source;
+  result->flags = 0;
+  return result;
+}
+
+/* Move to the next context.  Create one if there is none.  */
+static cpp_context *
+next_context (pfile)
+     cpp_reader *pfile;
+{
+  cpp_context *result = pfile->context->next;
+
+  if (result == 0)
     {
-      if (args[i].expanded)
-	free (args[i].expanded);
-      if (args[i].stringified)
-	free (args[i].stringified);
+      result = xnew (cpp_context);
+      result->prev = pfile->context;
+      result->next = 0;
+      pfile->context->next = result;
     }
+
+  pfile->context = result;
+  return result;
 }
 
-/* Subroutine of expand_arg to put the unexpanded tokens on the
-   context stack.  */
-static cpp_context *
-push_arg_context (pfile, arg)
+/* Push a list of pointers to tokens.  */
+static void
+push_ptoken_context (pfile, macro, first, count)
      cpp_reader *pfile;
-     macro_arg *arg;
+     cpp_macro *macro;
+     const cpp_token **first;
+     unsigned int count;
+{
+  cpp_context *context = next_context (pfile);
+
+  context->direct_p = false;
+  context->macro = macro;
+  context->first.ptoken = first;
+  context->last.ptoken = first + count;
+}
+
+/* Push a list of tokens.  */
+static void
+push_token_context (pfile, macro, first, count)
+     cpp_reader *pfile;
+     cpp_macro *macro;
+     const cpp_token *first;
+     unsigned int count;
 {
   cpp_context *context = next_context (pfile);
-  context->macro = 0;
-  context->list.first = arg->first;
-  context->list.limit = arg->first + arg->count + 1;
 
-  return context;
+  context->direct_p = true;
+  context->macro = macro;
+  context->first.token = first;
+  context->last.token = first + count;
 }
 
 static void
@@ -851,29 +902,39 @@ expand_arg (pfile, arg)
      cpp_reader *pfile;
      macro_arg *arg;
 {
-  cpp_token *token;
-  unsigned int capacity = 256;
+  unsigned int capacity;
+
+  arg->expanded_count = 0;
+  if (arg->count == 0)
+    return;
 
   /* Loop, reading in the arguments.  */
-  arg->expanded = (cpp_token *) xmalloc (capacity * sizeof (cpp_token));
+  capacity = 256;
+  arg->expanded = (const cpp_token **)
+    xmalloc (capacity * sizeof (cpp_token *));
 
-  push_arg_context (pfile, arg);
-  do
+  push_ptoken_context (pfile, NULL, arg->first, arg->count + 1);
+  for (;;)
     {
-      if (arg->expanded_count >= capacity)
+      const cpp_token *token;
+
+      if (arg->expanded_count + 1 >= capacity)
 	{
 	  capacity *= 2;
-	  arg->expanded = (cpp_token *)
-	    xrealloc (arg->expanded, capacity * sizeof (cpp_token));
+	  arg->expanded = (const cpp_token **)
+	    xrealloc (arg->expanded, capacity * sizeof (cpp_token *));
 	}
-      token = &arg->expanded[arg->expanded_count++];
-      cpp_get_token (pfile, token);
-    }
-  while (token->type != CPP_EOF);
 
-  arg->expanded_count--;
+      token = cpp_get_token (pfile);
+
+      if (token->type == CPP_EOF)
+	break;
+
+      arg->expanded[arg->expanded_count++] = token;
+    }
 
-  /* Pop the context we pushed.  */ 
+  /* Avoid the unlock_pools test of _cpp_pop_context.  Change this to
+     call _cpp_pop_context once we remove pool locking.  */
   pfile->context = pfile->context->prev;
 }
 
@@ -881,14 +942,13 @@ void
 _cpp_pop_context (pfile)
      cpp_reader *pfile;
 {
-  cpp_context *context = pfile->context;
+  /* Re-enable a macro when leaving its expansion.  */
+  if (pfile->context->macro)
+    pfile->context->macro->disabled = 0;
 
-  pfile->context = context->prev;
+  pfile->context = pfile->context->prev;
   if (!pfile->context->prev && !pfile->state.parsing_args)
     unlock_pools (pfile);
-
-  /* Re-enable a macro when leaving its expansion.  */
-  context->macro->disabled = 0;
 }
 
 /* Eternal routine to get a token.  Also used nearly everywhere
@@ -902,77 +962,75 @@ _cpp_pop_context (pfile)
    a directive inside a macro call, when at the end of a directive and
    state.in_directive is still 1, and at the end of argument
    pre-expansion.  */
-void
-cpp_get_token (pfile, token)
+const cpp_token *
+cpp_get_token (pfile)
      cpp_reader *pfile;
-     cpp_token *token;
 {
+  const cpp_token *result;
+
   for (;;)
     {
+      cpp_hashnode *node;
       cpp_context *context = pfile->context;
 
       /* Context->prev == 0 <=> base context.  */
       if (!context->prev)
-	*token = *_cpp_lex_token (pfile);
-      else if (context->list.first != context->list.limit)
+	result = _cpp_lex_token (pfile);
+      else if (context->first.token != context->last.token)
 	{
-	  *token = *context->list.first++;
-	  token->flags |= pfile->buffer->saved_flags;
-	  pfile->buffer->saved_flags = 0;
-	  /* PASTE_LEFT tokens can only appear in macro expansions.  */
-	  if (token->flags & PASTE_LEFT)
+	  if (context->direct_p)
+	    result = context->first.token++;
+	  else
+	    result = *context->first.ptoken++;
+
+	  if (result->flags & PASTE_LEFT)
 	    {
-	      /* Maintains position of original token.  */
-	      paste_all_tokens (pfile, token);
-	      pfile->buffer->saved_flags = AVOID_LPASTE;
+	      paste_all_tokens (pfile, result);
+	      if (pfile->state.in_directive)
+		continue;
+	      return padding_token (pfile, result);
 	    }
 	}
       else
 	{
-	  if (!context->macro)
-	    cpp_ice (pfile, "context->macro == 0");
-
-	  /* Avoid accidental paste at the end of a macro.  */
-	  pfile->buffer->saved_flags |= AVOID_LPASTE;
 	  _cpp_pop_context (pfile);
-	  continue;
+	  if (pfile->state.in_directive)
+	    continue;
+	  return &pfile->avoid_paste;
 	}
 
-      if (token->type != CPP_NAME)
+      if (result->type != CPP_NAME)
 	break;
 
+      node = result->val.node;
+
       /* Handle macros and the _Pragma operator.  */
-      if (token->val.node->type == NT_MACRO
-	  && !pfile->state.prevent_expansion
-	  && !(token->flags & NO_EXPAND))
+      if (node->type == NT_MACRO && !(result->flags & NO_EXPAND))
 	{
-	  cpp_hashnode *node = token->val.node;
-
 	  /* Macros invalidate controlling macros.  */
 	  pfile->mi_valid = false;
 
-	  if (node->flags & NODE_BUILTIN)
+	  if (!(node->flags & NODE_BUILTIN) && node->value.macro->disabled)
 	    {
-	      /* Maintains position of original token.  */
-	      builtin_macro (pfile, token);
-	      pfile->buffer->saved_flags = AVOID_LPASTE;
-	      break;
+	      /* Flag this token as always unexpandable.  */
+	      cpp_token *t = _cpp_temp_token (pfile);
+	      t->type = result->type;
+	      t->flags = result->flags | NO_EXPAND;
+	      t->val.str = result->val.str;
+	      result = t;
 	    }
-
-	  if (node->value.macro->disabled)
-	    token->flags |= NO_EXPAND;
-	  else if (enter_macro_context (pfile, node))
+	  else if (!pfile->state.prevent_expansion
+		   && enter_macro_context (pfile, node))
 	    {
-	      /* Pass AVOID_LPASTE and our PREV_WHITE to next token.  */
-	      pfile->buffer->saved_flags = ((token->flags & (PREV_WHITE | BOL))
-					    | AVOID_LPASTE);
-	      continue;
+	      if (pfile->state.in_directive)
+		continue;
+	      return padding_token (pfile, result);
 	    }
 	}
 
       /* Don't interpret _Pragma within directives.  The standard is
          not clear on this, but to me this makes most sense.  */
-      if (token->val.node != pfile->spec_nodes.n__Pragma
+      if (node != pfile->spec_nodes.n__Pragma
 	  || pfile->state.in_directive)
 	break;
 
@@ -980,6 +1038,8 @@ cpp_get_token (pfile, token)
          since this token came from either the lexer or a macro.  */
       _cpp_do__Pragma (pfile);
     }
+
+  return result;
 }
 
 /* Returns true if we're expanding an object-like macro that was
@@ -1000,11 +1060,8 @@ void
 cpp_scan_nooutput (pfile)
      cpp_reader *pfile;
 {
-  cpp_token token;
-
-  do
-    cpp_get_token (pfile, &token);
-  while (token.type != CPP_EOF);
+  while (cpp_get_token (pfile)->type != CPP_EOF)
+    ;
 }
 
 /* Step back one (or more) tokens.  Can only step mack more than 1 if
@@ -1031,7 +1088,10 @@ _cpp_backup_tokens (pfile, count)
     {
       if (count != 1)
 	abort ();
-      pfile->context->list.first--;
+      if (pfile->context->direct_p)
+	pfile->context->first.token--;
+      else
+	pfile->context->first.ptoken--;
     }
 }
 
@@ -1330,8 +1390,6 @@ _cpp_create_definition (pfile, node)
 	    }
 
 	  token[-1].flags |= PASTE_LEFT;
-	  /* Give it a PREV_WHITE for -dM etc.  */
-	  token->flags |= PREV_WHITE;
 	}
 
       token = lex_expansion_token (pfile, macro);
@@ -1340,13 +1398,6 @@ _cpp_create_definition (pfile, node)
   /* Don't count the CPP_EOF.  */
   macro->count--;
 
-  /* Clear the whitespace flag from the leading token, but put a space
-     in front of a leading # which might be used to fake a directive.  */
-  if (macro->expansion[0].type == CPP_HASH)
-    macro->expansion[0].flags |= PREV_WHITE;
-  else
-    macro->expansion[0].flags &= ~PREV_WHITE;
-
   /* Implement the macro-defined-to-itself optimisation.  */
   macro->disabled = (macro->count == 1 && !macro->fun_like
 		     && macro->expansion[0].type == CPP_NAME
diff --git a/gcc/cppmain.c b/gcc/cppmain.c
index 5451ccc31f4..a1b8b67f54c 100644
--- a/gcc/cppmain.c
+++ b/gcc/cppmain.c
@@ -32,6 +32,7 @@ struct printer
 {
   FILE *outf;			/* Stream to write to.  */
   const struct line_map *map;	/* Logical to physical line mappings.  */
+  const cpp_token *prev;	/* Previous token.  */
   unsigned int line;		/* Line currently being written.  */
   unsigned char printed;	/* Nonzero if something output at line.  */
 };
@@ -43,7 +44,7 @@ static void setup_callbacks PARAMS ((void));
 
 /* General output routines.  */
 static void scan_translation_unit PARAMS ((cpp_reader *));
-static void check_multiline_token PARAMS ((cpp_string *));
+static void check_multiline_token PARAMS ((const cpp_string *));
 static int dump_macro PARAMS ((cpp_reader *, cpp_hashnode *, void *));
 
 static void print_line PARAMS ((const struct line_map *, unsigned int,
@@ -144,6 +145,7 @@ do_preprocessing (argc, argv)
      cause a linemarker to be output by maybe_print_line.  */
   print.line = (unsigned int) -1;
   print.printed = 0;
+  print.prev = 0;
   print.map = 0;
   
   /* Open the output now.  We must do so even if no_output is on,
@@ -219,22 +221,43 @@ static void
 scan_translation_unit (pfile)
      cpp_reader *pfile;
 {
-  unsigned int index;
-  cpp_token tokens[2], *token;
+  bool avoid_paste = false;
+  const cpp_token *source = NULL;
 
-  for (index = 0;; index = 1 - index)
+  for (;;)
     {
-      token = &tokens[index];
-      cpp_get_token (pfile, token);
+      const cpp_token *token = cpp_get_token (pfile);
+
+      if (token->type == CPP_PADDING)
+	{
+	  avoid_paste = true;
+	  if (source == NULL
+	      || (!(source->flags & PREV_WHITE) && token->val.source == NULL))
+	    source = token->val.source;
+	  continue;
+	}
 
       if (token->type == CPP_EOF)
 	break;
 
-      if ((token->flags & (PREV_WHITE | AVOID_LPASTE | BOL)) == AVOID_LPASTE
-	  && cpp_avoid_paste (pfile, &tokens[1 - index], token))
-	token->flags |= PREV_WHITE;
+      /* Subtle logic to output a space if and only if necessary.  */
+      if (avoid_paste)
+	{
+	  if (source == NULL)
+	    source = token;
+	  if (source->flags & PREV_WHITE
+	      || (print.prev && cpp_avoid_paste (pfile, print.prev, token))
+	      || (print.prev == NULL && token->type == CPP_HASH))
+	    putc (' ', print.outf);
+	}
+      else if (token->flags & PREV_WHITE)
+	putc (' ', print.outf);
 
+      avoid_paste = false;
+      source = NULL;
+      print.prev = token;
       cpp_output_token (token, print.outf);
+
       if (token->type == CPP_STRING || token->type == CPP_WSTRING
 	  || token->type == CPP_COMMENT)
 	check_multiline_token (&token->val.str);
@@ -244,7 +267,7 @@ scan_translation_unit (pfile)
 /* Adjust print.line for newlines embedded in tokens.  */
 static void
 check_multiline_token (str)
-     cpp_string *str;
+     const cpp_string *str;
 {
   unsigned int i;
 
@@ -324,6 +347,7 @@ cb_line_change (pfile, token, parsing_args)
 
   maybe_print_line (print.map, token->line);
   print.printed = 1;
+  print.prev = 0;
 
   /* Supply enough spaces to put this token in its original column,
      one space per column greater than 2, since scan_translation_unit
diff --git a/gcc/doc/cpp.texi b/gcc/doc/cpp.texi
index 76d7b0cdef4..e8f7e9e7a68 100644
--- a/gcc/doc/cpp.texi
+++ b/gcc/doc/cpp.texi
@@ -1515,10 +1515,10 @@ token pasting.
 However, two tokens that don't together form a valid token cannot be
 pasted together.  For example, you cannot concatenate @code{x} with
 @code{+} in either order.  If you try, the preprocessor issues a warning
-and emits the two tokens as if they had been written next to each other.
-It is common to find unnecessary uses of @samp{##} in complex macros.
-If you get this warning, it is likely that you can simply remove the
-@samp{##}.
+and emits the two tokens.  Whether it puts white space between the
+tokens is undefined.  It is common to find unnecessary uses of @samp{##}
+in complex macros.  If you get this warning, it is likely that you can
+simply remove the @samp{##}.
 
 Both the tokens combined by @samp{##} could come from the macro body,
 but you could just as well write them as one token in the first place.
diff --git a/gcc/fix-header.c b/gcc/fix-header.c
index 24cbfef35e7..cf37a41a21d 100644
--- a/gcc/fix-header.c
+++ b/gcc/fix-header.c
@@ -661,12 +661,11 @@ read_scan_file (in_fname, argc, argv)
 		       /* from_stage3 */ true, 1);
       for (;;)
 	{
-	  cpp_token t;
+	  const cpp_token *t = cpp_get_token (scan_in);
 
-	  cpp_get_token (scan_in, &t);
-	  if (t.type == CPP_EOF)
+	  if (t->type == CPP_EOF)
 	    break;
-	  else if (cpp_ideq (&t, "_filbuf"))
+	  else if (cpp_ideq (t, "_filbuf"))
 	    seen_filbuf++;
 	}
 
diff --git a/gcc/scan-decls.c b/gcc/scan-decls.c
index fbe4e9d9ae1..cbd99004e9c 100644
--- a/gcc/scan-decls.c
+++ b/gcc/scan-decls.c
@@ -24,6 +24,7 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 #include "scan.h"
 
 static void skip_to_closing_brace PARAMS ((cpp_reader *));
+static const cpp_token *get_a_token PARAMS ((cpp_reader *));
 
 int brace_nesting = 0;
 
@@ -38,6 +39,19 @@ char extern_C_braces[20];
    prefixed by extern "C".  */
 int current_extern_C = 0;
 
+/* Get a token but skip padding.  */
+static const cpp_token *
+get_a_token (pfile)
+     cpp_reader *pfile;
+{
+  for (;;)
+    {
+      const cpp_token *result = cpp_get_token (pfile);
+      if (result->type != CPP_PADDING)
+	return result;
+    }
+}
+
 static void
 skip_to_closing_brace (pfile)
      cpp_reader *pfile;
@@ -45,11 +59,8 @@ skip_to_closing_brace (pfile)
   int nesting = 1;
   for (;;)
     {
-      cpp_token tok;
-      enum cpp_ttype token;
+      enum cpp_ttype token = get_a_token (pfile)->type;
 
-      cpp_get_token (pfile, &tok);
-      token = tok.type;
       if (token == CPP_EOF)
 	break;
       if (token == CPP_OPEN_BRACE)
@@ -88,16 +99,17 @@ scan_decls (pfile, argc, argv)
      char **argv ATTRIBUTE_UNUSED;
 {
   int saw_extern, saw_inline;
-  cpp_token token, prev_id;
+  cpp_token prev_id;
+  const cpp_token *token;
 
  new_statement:
-  cpp_get_token (pfile, &token);
+  token = get_a_token (pfile);
 
  handle_statement:
   current_extern_C = 0;
   saw_extern = 0;
   saw_inline = 0;
-  if (token.type == CPP_OPEN_BRACE)
+  if (token->type == CPP_OPEN_BRACE)
     {
       /* Pop an 'extern "C"' nesting level, if appropriate.  */
       if (extern_C_braces_length
@@ -106,24 +118,24 @@ scan_decls (pfile, argc, argv)
       brace_nesting--;
       goto new_statement;
     }
-  if (token.type == CPP_OPEN_BRACE)
+  if (token->type == CPP_OPEN_BRACE)
     {
       brace_nesting++;
       goto new_statement;
     }
 
-  if (token.type == CPP_EOF)
+  if (token->type == CPP_EOF)
     return 0;
 
-  if (token.type == CPP_SEMICOLON)
+  if (token->type == CPP_SEMICOLON)
     goto new_statement;
-  if (token.type != CPP_NAME)
+  if (token->type != CPP_NAME)
     goto new_statement;
 
   prev_id.type = CPP_EOF;
   for (;;)
     {
-      switch (token.type)
+      switch (token->type)
 	{
 	default:
 	  goto handle_statement;
@@ -138,7 +150,7 @@ scan_decls (pfile, argc, argv)
 	    {
 	      recognized_extern (&prev_id);
 	    }
-	  if (token.type == CPP_COMMA)
+	  if (token->type == CPP_COMMA)
 	    break;
 	  /* ... fall through ...  */
 	case CPP_OPEN_BRACE:  case CPP_CLOSE_BRACE:
@@ -155,27 +167,27 @@ scan_decls (pfile, argc, argv)
 	      int have_arg_list = 0;
 	      for (;;)
 		{
-		  cpp_get_token (pfile, &token);
-		  if (token.type == CPP_OPEN_PAREN)
+		  token = get_a_token (pfile);
+		  if (token->type == CPP_OPEN_PAREN)
 		    nesting++;
-		  else if (token.type == CPP_CLOSE_PAREN)
+		  else if (token->type == CPP_CLOSE_PAREN)
 		    {
 		      nesting--;
 		      if (nesting == 0)
 			break;
 		    }
-		  else if (token.type == CPP_EOF)
+		  else if (token->type == CPP_EOF)
 		    break;
-		  else if (token.type == CPP_NAME
-			   || token.type == CPP_ELLIPSIS)
+		  else if (token->type == CPP_NAME
+			   || token->type == CPP_ELLIPSIS)
 		    have_arg_list = 1;
 		}
-	      recognized_function (&prev_id, token.line,
+	      recognized_function (&prev_id, token->line,
 				   (saw_inline ? 'I'
 				    : in_extern_C_brace || current_extern_C
 				    ? 'F' : 'f'), have_arg_list);
-	      cpp_get_token (pfile, &token);
-	      if (token.type == CPP_OPEN_BRACE)
+	      token = get_a_token (pfile);
+	      if (token->type == CPP_OPEN_BRACE)
 		{
 		  /* skip body of (normally) inline function */
 		  skip_to_closing_brace (pfile);
@@ -184,28 +196,28 @@ scan_decls (pfile, argc, argv)
 
 	      /* skip a possible __attribute__ or throw expression after the
 		 parameter list */
-	      while (token.type != CPP_SEMICOLON && token.type != CPP_EOF)
-		cpp_get_token (pfile, &token);
+	      while (token->type != CPP_SEMICOLON && token->type != CPP_EOF)
+		token = get_a_token (pfile);
 	      goto new_statement;
 	    }
 	  break;
 	case CPP_NAME:
 	  /* "inline" and "extern" are recognized but skipped */
-	  if (cpp_ideq (&token, "inline"))
+	  if (cpp_ideq (token, "inline"))
 	    {
 	      saw_inline = 1;
 	    }
-	  else if (cpp_ideq (&token, "extern"))
+	  else if (cpp_ideq (token, "extern"))
 	    {
 	      saw_extern = 1;
-	      cpp_get_token (pfile, &token);
-	      if (token.type == CPP_STRING
-		  && token.val.str.len == 1
-		  && token.val.str.text[0] == 'C')
+	      token = get_a_token (pfile);
+	      if (token->type == CPP_STRING
+		  && token->val.str.len == 1
+		  && token->val.str.text[0] == 'C')
 		{
 		  current_extern_C = 1;
-		  cpp_get_token (pfile, &token);
-		  if (token.type == CPP_OPEN_BRACE)
+		  token = get_a_token (pfile);
+		  if (token->type == CPP_OPEN_BRACE)
 		    {
 		      brace_nesting++;
 		      extern_C_braces[extern_C_braces_length++]
@@ -218,9 +230,9 @@ scan_decls (pfile, argc, argv)
 	      break;
 	    }
 	  /* This may be the name of a variable or function.  */
-	  prev_id = token;
+	  prev_id = *token;
 	  break;
 	}
-      cpp_get_token (pfile, &token);
+      token = get_a_token (pfile);
     }
 }
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index b4981332a1a..f1ebfd165f5 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,11 @@
+2001-09-24  Neil Booth  <neil@daikokuya.demon.co.uk>
+
+	* gcc.dg/cpp/macro10.c: New test.
+	* gcc.dg/cpp/strify3.c: New test.
+	* gcc.dg/cpp/spacing1.c: Add tests.
+	* gcc.dg/cpp/19990703-1.c: Remove bogus test.
+	* gcc.dg/cpp/20000625-2.c: Fudge to pass.
+
 2001-09-24  DJ Delorie  <dj@redhat.com>
 
 	* gcc.c-torture/execute/20010924-1.c: New test.
diff --git a/gcc/testsuite/gcc.dg/cpp/19990703-1.c b/gcc/testsuite/gcc.dg/cpp/19990703-1.c
deleted file mode 100644
index c3ac2330055..00000000000
--- a/gcc/testsuite/gcc.dg/cpp/19990703-1.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/* { dg-do run } */
-
-/* Test of obscure case in token pasting in the preprocessor.
-   I can't think of any way to make this problem provoke a syntax error.
-   Based on a bug report by Manfred Hollstein.  */
-
-#include <string.h>
-
-#define SP1(x, y) SP2(x, y)
-#define SP2(x, y) SP3(x##y)
-#define SP3(x) #x
-#define MZ -0
-
-int
-main(void)
-{
-    char *x = SP1(0,MZ);  /* { dg-warning "valid preprocessing token" "" } */
-    char *y = "0-0";  /* should be the expansion of SP1(0,MZ) */
-
-    if(strcmp(x, y))
-	return 1;
-    else
-	return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/cpp/20000625-2.c b/gcc/testsuite/gcc.dg/cpp/20000625-2.c
index fb458984b8c..c9e3fe23b9c 100644
--- a/gcc/testsuite/gcc.dg/cpp/20000625-2.c
+++ b/gcc/testsuite/gcc.dg/cpp/20000625-2.c
@@ -8,7 +8,10 @@
 #define str(x) xstr(x)
 #define xstr(x) #x
 
-const char a[] = str(symbol_version(getrlimit, GLIBC_2.0));
+/* This testcase is bogus, as it testing undefined behaviour.  We can
+   get the behaviour GLIBC desires by removing the space before
+   GCLIB_2.0 in this line.  */
+const char a[] = str(symbol_version(getrlimit,GLIBC_2.0));
 /* { dg-warning "valid preprocessing token" "" { target *-*-* } 11 } */
 const char b[] = str(getrlimit@GLIBC_2.0);
 const char c[] = "getrlimit@GLIBC_2.0";
diff --git a/gcc/testsuite/gcc.dg/cpp/macro10.c b/gcc/testsuite/gcc.dg/cpp/macro10.c
new file mode 100644
index 00000000000..20d4911e648
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/macro10.c
@@ -0,0 +1,25 @@
+/* Copyright (C) 2001 Free Software Foundation, Inc.  */
+
+/* { dg-do preprocess } */
+
+/* Source: Neil Booth, 23 Sep 2001.
+
+   A tricky, pathological corner case we used to get wrong.  Expansion
+   should go as follows.  The asterisk indicates the token has "blue
+   paint" can no longer be macro expanded.  We used to lose that
+   information when parsing arguments and dropping to the lexer to get
+   the ')'.
+
+   foo )
+   bar foo* )
+   func (foo* )
+   foo*   
+
+   If we try and expand the final foo, we get an "unterminated
+   argument list invoking macro <func>" error.  If we do the right
+   thing and leave it as is, no diagnostics are emitted.  */
+
+#define func(x) x
+#define bar func(
+#define foo bar foo
+foo )
diff --git a/gcc/testsuite/gcc.dg/cpp/spacing1.c b/gcc/testsuite/gcc.dg/cpp/spacing1.c
index 55a8f967c81..9a3933b7308 100644
--- a/gcc/testsuite/gcc.dg/cpp/spacing1.c
+++ b/gcc/testsuite/gcc.dg/cpp/spacing1.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2000 Free Software Foundation, Inc.  */
+/* Copyright (C) 2000, 2001 Free Software Foundation, Inc.  */
 
 /* { dg-do preprocess } */
 
@@ -8,20 +8,24 @@
    not be a macro invocation.  Also, multiple macro invocations spread
    across many lines.
 
-   Neil Booth, 1 Dec 2000.  */
+   Neil Booth, 1 Dec 2000, 23 Sep 2001.  */
 
 #define str(x) #x
 #define f(x) x
+#define glue(x, y) x ## y
+#define EMPTY
 
 /* The correct output is shown here.  Note the spaces, and the way
    everything after the invocation of f appears on the same line.
 
+ 44 ;
 f
 bar
 g "1 2" bam baz
 
 */
 
+glue (EMPTY 4, 4) EMPTY;
 f
 bar
 f (g) str
@@ -33,9 +37,10 @@ f (g) str
 
 /*
    { dg-final { if ![file exists spacing1.i] { return }                   } }
+   { dg-final { if \{ [grep spacing1.i " 44 ;"] != "" \}  \{              } }
    { dg-final { if \{ [grep spacing1.i "f.*bar"] == "" \} \{              } }
    { dg-final { if \{ [grep spacing1.i "^bar"] != "" \}   \{              } }
    { dg-final { if \{ [grep spacing1.i "g \"1 2\" bam baz"] != "" \} \{   } }
-   { dg-final { return \} \} \}                                           } }
+   { dg-final { return \} \} \} \}                                        } }
    { dg-final { fail "spacing1.c: spacing and new-line preservation"      } }
 */
diff --git a/gcc/testsuite/gcc.dg/cpp/strify3.c b/gcc/testsuite/gcc.dg/cpp/strify3.c
new file mode 100644
index 00000000000..5d76b5e750f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/strify3.c
@@ -0,0 +1,29 @@
+/* Copyright (C) 2001 Free Software Foundation, Inc.  */
+
+/* { dg-do run } */
+
+/* Tests we stringify without inserting a space.  GCC 2.95.x and
+   earlier would insert a bogus space before bar in the string, simply
+   because a space was there in the invocation.
+
+   Neil Booth, 24 Sep 2001.  */
+
+extern int strcmp (const char *, const char *);
+extern int puts (const char *);
+extern void abort (void);
+#define err(str) do { puts(str); abort(); } while (0)
+
+#define str(x) #x
+#define xstr(x) str(x)
+#define glibc_hack(x, y) x@y
+
+int main (int argc, char *argv[])
+{
+  /* The space before "bar" here is vital.  */
+  char a[] = xstr(glibc_hack(foo, bar));
+
+  if (strcmp (a, "foo@bar"))
+    err ("stringification without spaces");
+
+  return 0;
+}