BR3288901: Relax concat rules in preprocessor code

We simply allow the following terminals to be concat'ed
if they are written without space or any other separator
inbetween.

        a := id | preproc-id | number | float | other
        b := id | preproc-id | number | float | other

        if match(a,b):
                s := concat(a,b)
                re-tokenize(s)

Basically it means it's up to code author to write
preproc code a way the sane production appears.

Some notes.

1) We don't concat strings.

2) The 'weirdpaste' test fails now because with relaxed
   rules it works as needed and was borken before.

   The lacmus snippet is

        %define N 1e%++%+ 5
        dd N, 1e+5

Previously the output was

        dd 1e+%+ 5, 1e+5

which is wrong since we have explicit concat here
with %+ operator. The new code production is correct
and looks like

        dd 1e+5, 1e+5

as expected.

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>

1 files changed, 23 insertions, 27 deletions

diff --git a/preproc.c b/preproc.c
index 5fdeab26..29d5aff9 100644
--- a/preproc.c
+++ b/preproc.c
@@ -337,6 +337,22 @@ enum {
 /* max reps */
 #define REP_LIMIT ((INT64_C(1) << 62))
 
+const struct tokseq_match pp_concat_match[] = {
+    {
+        PP_CONCAT_MASK(TOK_ID)          |
+        PP_CONCAT_MASK(TOK_PREPROC_ID)  |
+        PP_CONCAT_MASK(TOK_NUMBER)      |
+        PP_CONCAT_MASK(TOK_FLOAT)       |
+        PP_CONCAT_MASK(TOK_OTHER),
+
+        PP_CONCAT_MASK(TOK_ID)          |
+        PP_CONCAT_MASK(TOK_PREPROC_ID)  |
+        PP_CONCAT_MASK(TOK_NUMBER)      |
+        PP_CONCAT_MASK(TOK_FLOAT)       |
+        PP_CONCAT_MASK(TOK_OTHER)
+    }
+};
+
 /*
  * Condition codes. Note that we use c_ prefix not C_ because C_ is
  * used in nasm.h for the "real" condition codes. At _this_ level,
@@ -4304,23 +4320,10 @@ static Token *expand_mmac_params(Token * tline)
     }
     *tail = NULL;
 
-    if (changed) {
-        const struct tokseq_match t[] = {
-            {
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_FLOAT),          /* head */
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_NUMBER)      |
-                PP_CONCAT_MASK(TOK_FLOAT)       |
-                PP_CONCAT_MASK(TOK_OTHER)           /* tail */
-            },
-            {
-                PP_CONCAT_MASK(TOK_NUMBER),         /* head */
-                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
-            }
-        };
-        paste_tokens(&thead, t, ARRAY_SIZE(t), false);
-    }
+    if (changed)
+        paste_tokens(&thead, pp_concat_match,
+                     ARRAY_SIZE(pp_concat_match),
+                     false);
 
     return thead;
 }
@@ -4632,16 +4635,9 @@ again:
      * them (without white spaces in between).
      */
     if (expanded) {
-        const struct tokseq_match t[] = {
-            {
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_PREPROC_ID),     /* head */
-                PP_CONCAT_MASK(TOK_ID)          |
-                PP_CONCAT_MASK(TOK_PREPROC_ID)  |
-                PP_CONCAT_MASK(TOK_NUMBER)          /* tail */
-            }
-        };
-        if (paste_tokens(&thead, t, ARRAY_SIZE(t), true)) {
+        if (paste_tokens(&thead, pp_concat_match,
+                         ARRAY_SIZE(pp_concat_match),
+                         true)) {
             /*
              * If we concatenated something, *and* we had previously expanded
              * an actual macro, scan the lines again for macros...