Handle weird cases of token pasting

Especially when token pasting involves floating-point numbers, we can
have some really strange effects from token pasting: for example,
pasting the two tokens "xyzzy" and "1e+10" ends up with *three*
tokens: "xyzzy1e" "+" "10".  The easiest way to deal with this is to
explicitly combine the string and then run tokenize() on it.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 54 insertions, 13 deletions

diff --git a/preproc.c b/preproc.c
index 0579b0f4..fd621f51 100644
--- a/preproc.c
+++ b/preproc.c
@@ -159,7 +159,7 @@ enum pp_token_type {
     TOK_NUMBER, TOK_FLOAT, TOK_SMAC_END, TOK_OTHER,
     TOK_INTERNAL_STRING,
     TOK_PREPROC_Q, TOK_PREPROC_QQ,
-    TOK_INDIRECT,	   	/* %[...] */
+    TOK_INDIRECT,		/* %[...] */
     TOK_SMAC_PARAM,		/* MUST BE LAST IN THE LIST!!! */
     TOK_MAX = INT_MAX		/* Keep compiler from reducing the range */
 };
@@ -3467,29 +3467,56 @@ static Token *expand_mmac_params(Token * tline)
     *tail = NULL;
 
     /* Now handle token pasting... */
-    t = thead;
-    while (t && (tt = t->next)) {
+    tail = &thead;
+    while ((t = *tail) && (tt = t->next)) {
         switch (t->type) {
         case TOK_WHITESPACE:
             if (tt->type == TOK_WHITESPACE) {
                 t->next = delete_Token(tt);
             } else {
-		t = tt;
+		tail = &t->next;
 	    }
             break;
         case TOK_ID:
         case TOK_NUMBER:
-            if (tt->type == t->type || tt->type == TOK_NUMBER) {
-                char *tmp = nasm_strcat(t->text, tt->text);
-                nasm_free(t->text);
-                t->text = tmp;
-                t->next = delete_Token(tt);
-            } else {
-		t = tt;
+	case TOK_FLOAT:
+	{
+	    size_t len = 0;
+	    char *tmp, *p;
+
+	    while (tt &&
+		   (tt->type == TOK_ID || tt->type == TOK_NUMBER ||
+		    tt->type == TOK_FLOAT)) {
+		len += strlen(tt->text);
+		tt = tt->next;
 	    }
-            break;
+
+	    /* Now tt points to the first token after the potential
+	       paste area... */
+	    if (tt != t->next) {
+		/* We have at least two tokens... */
+		len += strlen(t->text);
+		p = tmp = nasm_malloc(len+1);
+
+		while (t != tt) {
+		    strcpy(p, t->text);
+		    p = strchr(p, '\0');
+		    t = delete_Token(t);
+		}
+
+		t = *tail = tokenize(tmp);
+		nasm_free(tmp);
+
+		while (t->next)
+		    t = t->next;
+
+		t->next = tt;	/* Attach the remaining token chain */
+	    }
+	    tail = &t->next;
+	    break;
+	}
 	default:
-	    t = tt;
+	    tail = &t->next;
 	    break;
         }
     }
diff --git a/test/weirdpaste.asm b/test/weirdpaste.asm
new file mode 100644
index 00000000..c6e98150
--- /dev/null
+++ b/test/weirdpaste.asm
@@ -0,0 +1,14 @@
+;Testname=preproc; Arguments=-E; Files=stdout stderr
+;Testname=bin; Arguments=-fbin -oweirdpaste.bin; Files=stdout stderr weirdpaste.bin
+
+	%define foo xyzzy
+%define bar 1e+10
+
+%define xyzzy1e 15
+
+%macro dx 2
+%assign	xx %1%2
+	dw xx
+%endmacro
+
+	dx foo, bar