Fix command expansion and parsing.

Tox testenv commands are parsed to expand variable substitutions and
construct the argv list that will be passed to exec. Prior to this
commit this parsing ate quotes surrounding variables and treated
multiword variables as single argv items. Neither behavior was correct.
To fix this create the expanded command before handing it off to shlex
to do the tokenization of the argv list. Doing the parsing in this
order ensures it is correct.

2 files changed, 25 insertions, 20 deletions

diff --git a/tests/test_config.py b/tests/test_config.py
index b8d0aa36..7ffc4f5c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -278,7 +278,7 @@ class TestIniParser:
         #    "reader.getargvlist('section', 'key1')")
         assert reader.getargvlist('section', 'key1') == []
         x = reader.getargvlist("section", "key2")
-        assert x == [["cmd1", "with space", "grr"],
+        assert x == [["cmd1", "with", "space", "grr"],
                      ["cmd2", "grr"]]
 
     def test_argvlist_windows_escaping(self, tmpdir, newconfig):
@@ -304,7 +304,7 @@ class TestIniParser:
         #    "reader.getargvlist('section', 'key1')")
         assert reader.getargvlist('section', 'key1') == []
         x = reader.getargvlist("section", "key2")
-        assert x == [["cmd1", "with space", "grr"]]
+        assert x == [["cmd1", "with", "space", "grr"]]
 
 
     def test_argvlist_quoting_in_command(self, tmpdir, newconfig):
diff --git a/tox/_config.py b/tox/_config.py
index 3644d8bd..7639f63c 100644
--- a/tox/_config.py
+++ b/tox/_config.py
@@ -527,30 +527,35 @@ class IniReader:
     def _processcommand(self, command):
         posargs = getattr(self, "posargs", None)
 
-        # special treat posargs which might contain multiple arguments
-        # in their defaults
+        # Iterate through each word of the command substituting as
+        # appropriate to construct the new command string. This
+        # string is then broken up into exec argv components using
+        # shlex.
         newcommand = ""
         for word in CommandParser(command).words():
-            if word.startswith("{posargs:") and word.endswith("}"):
+            if word == "{posargs}" or word == "[]":
                 if posargs:
-                    word = "{posargs}"
+                    newcommand += " ".join(posargs)
+                continue
+            elif word.startswith("{posargs:") and word.endswith("}"):
+                if posargs:
+                    newcommand += " ".join(posargs)
+                    continue
                 else:
                     word = word[9:-1]
-            newcommand += word
-
-        # now we can properly parse the command
-        argv = []
-        for arg in shlex.split(newcommand):
-            if arg in ('[]', "{posargs}"):
-                if posargs:
-                    argv.extend(posargs)
-                continue
             new_arg = ""
-            for word in CommandParser(arg).words():
-                new_word = self._replace(word)
-                new_word = self._replace(new_word)
-                new_arg += new_word
-            argv.append(new_arg)
+            new_word = self._replace(word)
+            new_word = self._replace(new_word)
+            new_arg += new_word
+            newcommand += new_arg
+
+        # Construct shlex object that will not escape any values,
+        # use all values as is in argv.
+        shlexer = shlex.shlex(newcommand, posix=True)
+        shlexer.whitespace_split = True
+        shlexer.escape = ''
+        shlexer.commenters = ''
+        argv = list(shlexer)
         return argv
 
     def getargv(self, section, name, default=None, replace=True):