allow passing unknown args to provisioned tox (#1282)

tox ignores unknown CLI arguments when provisioning is on and outside
of the provisioned environment (allowing provisioning arguments to be
forwarded freely)

8 files changed, 85 insertions, 3 deletions

diff --git a/docs/changelog/1270.bugfix.rst b/docs/changelog/1270.bugfix.rst
new file mode 100644
index 00000000..38afe066
--- /dev/null
+++ b/docs/changelog/1270.bugfix.rst
@@ -0,0 +1,2 @@
+tox ignores unknown CLI arguments when provisioning is on and outside of the provisioned environment (allowing
+provisioning arguments to be forwarded freely) - by :user:`gaborbernat`
diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py
index 8180705b..70aa45a3 100644
--- a/src/tox/config/__init__.py
+++ b/src/tox/config/__init__.py
@@ -45,6 +45,8 @@ Instead create a hookimpl in your code with:
 default_factors = tox.PYTHON.DEFAULT_FACTORS
 """DEPRECATED MOVE - please update to new location."""
 
+WITHIN_PROVISION = os.environ.get(str("TOX_PROVISION")) == "1"
+
 
 def get_plugin_manager(plugins=()):
     # initialize plugin manager
@@ -114,8 +116,11 @@ class Parser:
         assert hasattr(obj, "postprocess")
         self._testenv_attr.append(obj)
 
-    def parse_cli(self, args):
-        return self.argparser.parse_args(args)
+    def parse_cli(self, args, strict=False):
+        if strict or WITHIN_PROVISION:
+            return self.argparser.parse_args(args)
+        else:
+            return self.argparser.parse_known_args(args)[0]
 
     def _format_help(self):
         return self.argparser.format_help()
@@ -1106,6 +1111,9 @@ class ParseIni(object):
             env_config.deps = deps
             config.envconfigs[config.provision_tox_env] = env_config
             raise tox.exception.MissingRequirement(config)
+        # if provisioning is not on, now we need do a strict argument evaluation
+        # raise on unknown args
+        self.config._parser.parse_cli(args=self.config.args, strict=True)
 
     @staticmethod
     def ensure_requires_satisfied(config, requires, min_version):
diff --git a/src/tox/session/commands/provision.py b/src/tox/session/commands/provision.py
index 3f0ac619..21825fc2 100644
--- a/src/tox/session/commands/provision.py
+++ b/src/tox/session/commands/provision.py
@@ -1,6 +1,8 @@
 """In case the tox environment is not correctly setup provision it and delegate execution"""
 from __future__ import absolute_import, unicode_literals
 
+import os
+
 from tox.exception import InvocationError
 
 
@@ -9,7 +11,9 @@ def provision_tox(provision_venv, args):
     with provision_venv.new_action("provision") as action:
         provision_args = [str(provision_venv.envconfig.envpython), "-m", "tox"] + args
         try:
-            action.popen(provision_args, redirect=False, report_fail=False)
+            env = os.environ.copy()
+            env[str("TOX_PROVISION")] = str("1")
+            action.popen(provision_args, redirect=False, report_fail=False, env=env)
             return 0
         except InvocationError as exception:
             return exception.exit_code
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/unit/__init__.py
diff --git a/tests/unit/session/plugin/a/__init__.py b/tests/unit/session/plugin/a/__init__.py
new file mode 100644
index 00000000..dbe24639
--- /dev/null
+++ b/tests/unit/session/plugin/a/__init__.py
@@ -0,0 +1,8 @@
+import pluggy
+
+hookimpl = pluggy.HookimplMarker("tox")
+
+
+@hookimpl
+def tox_addoption(parser):
+    parser.add_argument("--option", choices=["a", "b"], default="a", required=False)
diff --git a/tests/unit/session/plugin/setup.cfg b/tests/unit/session/plugin/setup.cfg
new file mode 100644
index 00000000..285b17de
--- /dev/null
+++ b/tests/unit/session/plugin/setup.cfg
@@ -0,0 +1,10 @@
+[metadata]
+name = plugin
+description = test stuff
+version = 0.1
+[options]
+zip_safe = True
+packages = find:
+
+[options.entry_points]
+tox = plugin = a
diff --git a/tests/unit/session/plugin/setup.py b/tests/unit/session/plugin/setup.py
new file mode 100644
index 00000000..60684932
--- /dev/null
+++ b/tests/unit/session/plugin/setup.py
@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()
diff --git a/tests/unit/session/test_provision.py b/tests/unit/session/test_provision.py
index ae4b4abc..6408dc65 100644
--- a/tests/unit/session/test_provision.py
+++ b/tests/unit/session/test_provision.py
@@ -1,5 +1,8 @@
+import shutil
+import subprocess
 import sys
 
+import py
 import pytest
 
 from tox.exception import BadRequirement, MissingRequirement
@@ -94,3 +97,47 @@ def test_provision_bad_requires(newconfig, capsys, monkeypatch):
     out, err = capsys.readouterr()
     assert "ERROR: failed to parse RequirementParseError" in out
     assert not err
+
+
+@pytest.fixture()
+def plugin(monkeypatch, tmp_path):
+    dest = tmp_path / "a"
+    shutil.copytree(str(py.path.local(__file__).dirpath().join("plugin")), str(dest))
+    subprocess.check_output([sys.executable, "setup.py", "egg_info"], cwd=str(dest))
+    monkeypatch.setenv(str("PYTHONPATH"), str(dest))
+
+
+def test_provision_cli_args_ignore(cmd, initproj, monkeypatch, plugin):
+    import tox.config
+    import tox.session
+
+    prev_ensure = tox.config.ParseIni.ensure_requires_satisfied
+
+    @staticmethod
+    def ensure_requires_satisfied(config, requires, min_version):
+        result = prev_ensure(config, requires, min_version)
+        config.run_provision = True
+        return result
+
+    monkeypatch.setattr(
+        tox.config.ParseIni, "ensure_requires_satisfied", ensure_requires_satisfied
+    )
+    prev_get_venv = tox.session.Session.getvenv
+
+    def getvenv(self, name):
+        venv = prev_get_venv(self, name)
+        venv.envconfig.envdir = py.path.local(sys.executable).dirpath().dirpath()
+        venv.setupenv = lambda: True
+        venv.finishvenv = lambda: True
+        return venv
+
+    monkeypatch.setattr(tox.session.Session, "getvenv", getvenv)
+    initproj("test-0.1", {"tox.ini": "[tox]"})
+    result = cmd("-a", "--option", "b")
+    result.assert_success(is_run_test_env=False)
+
+
+def test_provision_cli_args_not_ignored_if_provision_false(cmd, initproj):
+    initproj("test-0.1", {"tox.ini": "[tox]"})
+    result = cmd("-a", "--option", "b")
+    result.assert_fail(is_run_test_env=False)