preproc.c: Restore concat rules on context local variables

This is a backport of commits

8dcbbd7af0d6d07b455de0b6460dca6db6113553
575d4289c9b1fb47774cb79764a24899a69a8d52

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>

1 files changed, 95 insertions, 89 deletions

diff --git a/preproc.c b/preproc.c
index af466637..d7a96d82 100644
--- a/preproc.c
+++ b/preproc.c
@@ -212,6 +212,13 @@ enum pp_token_type {
     TOK_MAX = INT_MAX       /* Keep compiler from reducing the range */
 };
 
+#define PP_CONCAT_MASK(x) (1 << (x))
+
+struct tokseq_match {
+    int mask_head;
+    int mask_tail;
+};
+
 struct Token {
     Token *next;
     char *text;
@@ -3642,12 +3649,14 @@ static int find_cc(Token * t)
     return i;
 }
 
-static bool paste_tokens(Token **head, bool handle_paste_tokens)
+static bool paste_tokens(Token **head, const struct tokseq_match *m,
+                         int mnum, bool handle_paste_tokens)
 {
     Token **tail, *t, *tt;
     Token **paste_head;
     bool did_paste = false;
     char *tmp;
+    int i;
 
     /* Now handle token pasting... */
     paste_head = NULL;
@@ -3663,51 +3672,6 @@ static bool paste_tokens(Token **head, bool handle_paste_tokens)
                 tail = &t->next;
             }
             break;
-        case TOK_ID:
-        case TOK_PREPROC_ID:
-        case TOK_NUMBER:
-        case TOK_FLOAT:
-        {
-            size_t len = 0;
-            char *tmp, *p;
-
-            while (tt && (tt->type == TOK_ID || tt->type == TOK_PREPROC_ID ||
-                          tt->type == TOK_NUMBER || tt->type == TOK_FLOAT ||
-                          tt->type == TOK_OTHER)) {
-                len += strlen(tt->text);
-                tt = tt->next;
-            }
-
-            /*
-             * Now tt points to the first token after
-             * the potential paste area...
-             */
-            if (tt != t->next) {
-                /* We have at least two tokens... */
-                len += strlen(t->text);
-                p = tmp = nasm_malloc(len+1);
-
-                while (t != tt) {
-                    strcpy(p, t->text);
-                    p = strchr(p, '\0');
-                    t = delete_Token(t);
-                }
-
-                t = *tail = tokenize(tmp);
-                nasm_free(tmp);
-
-                while (t->next) {
-                    tail = &t->next;
-                    t = t->next;
-                }
-                t->next = tt;   /* Attach the remaining token chain */
-
-                did_paste = true;
-            }
-            paste_head = tail;
-            tail = &t->next;
-            break;
-        }
         case TOK_PASTE:         /* %+ */
             if (handle_paste_tokens) {
                 /* Zap %+ and whitespace tokens to the right */
@@ -3721,7 +3685,6 @@ static bool paste_tokens(Token **head, bool handle_paste_tokens)
                 tt = t->next;
                 while (tok_type_(tt, TOK_WHITESPACE))
                     tt = t->next = delete_Token(tt);
-
                 if (tt) {
                     tmp = nasm_strcat(t->text, tt->text);
                     delete_Token(t);
@@ -3741,9 +3704,55 @@ static bool paste_tokens(Token **head, bool handle_paste_tokens)
             }
             /* else fall through */
         default:
-            tail = &t->next;
-            if (!tok_type_(t->next, TOK_WHITESPACE))
-                paste_head = tail;
+            /*
+             * Concatenation of tokens might look nontrivial
+             * but in real it's pretty simple -- the caller
+             * prepares the masks of token types to be concatenated
+             * and we simply find matched sequences and slip
+             * them together
+             */
+            for (i = 0; i < mnum; i++) {
+                if (PP_CONCAT_MASK(t->type) & m[i].mask_head) {
+                    size_t len = 0;
+                    char *tmp, *p;
+
+                    while (tt && (PP_CONCAT_MASK(tt->type) & m[i].mask_tail)) {
+                        len += strlen(tt->text);
+                        tt = tt->next;
+                    }
+
+                    /*
+                     * Now tt points to the first token after
+                     * the potential paste area...
+                     */
+                    if (tt != t->next) {
+                        /* We have at least two tokens... */
+                        len += strlen(t->text);
+                        p = tmp = nasm_malloc(len+1);
+                        while (t != tt) {
+                            strcpy(p, t->text);
+                            p = strchr(p, '\0');
+                            t = delete_Token(t);
+                        }
+                        t = *tail = tokenize(tmp);
+                        nasm_free(tmp);
+                        while (t->next) {
+                            tail = &t->next;
+                            t = t->next;
+                        }
+                        t->next = tt;   /* Attach the remaining token chain */
+                        did_paste = true;
+                    }
+                    paste_head = tail;
+                    tail = &t->next;
+                    break;
+                }
+            }
+            if (i >= mnum) {    /* no match */
+                tail = &t->next;
+                if (!tok_type_(t->next, TOK_WHITESPACE))
+                    paste_head = tail;
+            }
             break;
         }
     }
@@ -3977,35 +3986,6 @@ static Token *expand_mmac_params(Token * tline)
             }
             delete_Token(t);
             changed = true;
-        } else if (tline->type == TOK_PREPROC_ID                &&
-                   tline->text[0] == '%'                        &&
-                   tline->text[1] == '$'                        &&
-                   !tok_type_(tline->next, TOK_WHITESPACE)      &&
-                   (tok_type_(tline->next, TOK_ID)              ||
-                    tok_type_(tline->next, TOK_PREPROC_ID)      ||
-                    tok_type_(tline->next, TOK_NUMBER)          ||
-                    tok_type_(tline->next, TOK_OTHER)           ||
-                    tok_type_(tline->next, TOK_FLOAT))) {
-            /*
-             * In a sake of backward compatibility we allow
-             * to expand local single macro that early before
-             * pasting token code have place
-             *
-             * NOTE: that new code MUST use %+ macro to obtain
-             * same result
-             */
-            t = tline;
-            tline = tline->next;
-            tt = tokenize(t->text);
-            tt = expand_smacro(tt);
-            *tail = tt;
-            while (tt) {
-                tt->a.mac = NULL;
-                tail = &tt->next;
-                tt = tt->next;
-            }
-            delete_Token(t);
-            changed = true;
         } else {
             t = *tail = tline;
             tline = tline->next;
@@ -4015,8 +3995,23 @@ static Token *expand_mmac_params(Token * tline)
     }
     *tail = NULL;
 
-    if (changed)
-        paste_tokens(&thead, false);
+    if (changed) {
+        const struct tokseq_match t[] = {
+            {
+                PP_CONCAT_MASK(TOK_ID)          |
+                PP_CONCAT_MASK(TOK_FLOAT),          /* head */
+                PP_CONCAT_MASK(TOK_ID)          |
+                PP_CONCAT_MASK(TOK_NUMBER)      |
+                PP_CONCAT_MASK(TOK_FLOAT)       |
+                PP_CONCAT_MASK(TOK_OTHER)           /* tail */
+            },
+            {
+                PP_CONCAT_MASK(TOK_NUMBER),         /* head */
+                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
+            }
+        };
+        paste_tokens(&thead, t, ARRAY_SIZE(t), false);
+    }
 
     return thead;
 }
@@ -4327,14 +4322,25 @@ again:
      * Also we look for %+ tokens and concatenate the tokens before and after
      * them (without white spaces in between).
      */
-    if (expanded && paste_tokens(&thead, true)) {
-        /*
-         * If we concatenated something, *and* we had previously expanded
-         * an actual macro, scan the lines again for macros...
-         */
-        tline = thead;
-        expanded = false;
-        goto again;
+    if (expanded) {
+        const struct tokseq_match t[] = {
+            {
+                PP_CONCAT_MASK(TOK_ID)          |
+                PP_CONCAT_MASK(TOK_PREPROC_ID),     /* head */
+                PP_CONCAT_MASK(TOK_ID)          |
+                PP_CONCAT_MASK(TOK_PREPROC_ID)  |
+                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
+            }
+        };
+        if (paste_tokens(&thead, t, ARRAY_SIZE(t), true)) {
+            /*
+             * If we concatenated something, *and* we had previously expanded
+             * an actual macro, scan the lines again for macros...
+             */
+            tline = thead;
+            expanded = false;
+            goto again;
+        }
     }
 
 err: