Merge pull request #1403 from timothycrosley/separate-integration-test

Separate integration test

34 files changed, 155 insertions, 16 deletions

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 00000000..28310311
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,35 @@
+name: Integration
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: pip cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: integration-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            integration-pip-
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade poetry
+          poetry install
+
+      - name: Test integration
+        run: ./scripts/test_integration.sh
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 71788b54..ef93a1ee 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -57,7 +57,7 @@ jobs:
       - name: Test
         shell: bash
         run: |
-          poetry run pytest tests/ -s --cov=isort/ --cov-report=term-missing ${@-}
+          poetry run pytest tests/unit/ -s --cov=isort/ --cov-report=term-missing ${@-}
           poetry run coverage xml
       - name: Report Coverage
         if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.8'
diff --git a/.isort.cfg b/.isort.cfg
index 3f3b28ad..567d1abd 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,4 +1,4 @@
 [settings]
 profile=hug
 src_paths=isort,test
-skip=tests/example_crlf_file.py
+skip=tests/unit/example_crlf_file.py
diff --git a/scripts/test.sh b/scripts/test.sh
index a0659852..5aa1c764 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -2,5 +2,5 @@
 set -euxo pipefail
 
 ./scripts/lint.sh
-poetry run pytest tests/ -s --cov=isort/ --cov-report=term-missing ${@-}
+poetry run pytest tests/unit/ -s --cov=isort/ --cov-report=term-missing ${@-}
 poetry run coverage html
diff --git a/scripts/test_integration.sh b/scripts/test_integration.sh
new file mode 100755
index 00000000..8bbcc5e7
--- /dev/null
+++ b/scripts/test_integration.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euxo pipefail
+
+poetry run pytest tests/integration/ -s
diff --git a/setup.cfg b/setup.cfg
index 770db0e1..8129063f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,4 +25,4 @@ exclude = _vendored
 per-file-ignores =
     isort/__init__.py:F401
     isort/stdlibs/__init__.py:F401
-    tests/example_crlf_file.py:F401
+    tests/unit/example_crlf_file.py:F401
diff --git a/tests/test_hypothesmith.py b/tests/integration/test_hypothesmith.py
index 94964cee..94964cee 100644
--- a/tests/test_hypothesmith.py
+++ b/tests/integration/test_hypothesmith.py
diff --git a/tests/integration/test_projects_using_isort.py b/tests/integration/test_projects_using_isort.py
new file mode 100644
index 00000000..f1a1f46e
--- /dev/null
+++ b/tests/integration/test_projects_using_isort.py
@@ -0,0 +1,110 @@
+"""Tests a projects that use isort to see if any differences are found between
+their current imports and what isort suggest on the develop branch.
+This is an important early warning signal of regressions.
+
+NOTE: If you use isort within a public repository, please feel empowered to add your project here!
+It is important to isort that as few regressions as possible are experienced by our users.
+Having your project tested here is the most sure way to keep those regressions form ever happening.
+"""
+from subprocess import check_call
+
+from isort.main import main
+
+
+def test_django(tmpdir):
+    check_call(["git", "clone", "https://github.com/django/django.git", str(tmpdir)])
+    isort_target_dirs = [
+        str(target_dir) for target_dir in (tmpdir / "django", tmpdir / "tests", tmpdir / "scripts")
+    ]
+    main(["--check-only", "--diff", *isort_target_dirs])
+
+
+def test_plone(tmpdir):
+    check_call(
+        ["git", "clone", "https://github.com/plone/plone.app.multilingualindexes.git", str(tmpdir)]
+    )
+    main(["--check-only", "--diff", str(tmpdir / "src")])
+
+
+def test_pandas(tmpdir):
+    check_call(["git", "clone", "https://github.com/pandas-dev/pandas.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir / "pandas"), "--skip", "__init__.py"])
+
+
+def test_fastapi(tmpdir):
+    check_call(["git", "clone", "https://github.com/tiangolo/fastapi.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir / "fastapi")])
+
+
+def test_zulip(tmpdir):
+    check_call(["git", "clone", "https://github.com/zulip/zulip.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir), "--skip", "__init__.pyi"])
+
+
+def test_habitat_lab(tmpdir):
+    check_call(["git", "clone", "https://github.com/facebookresearch/habitat-lab.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir)])
+
+
+def test_tmuxp(tmpdir):
+    check_call(["git", "clone", "https://github.com/tmux-python/tmuxp.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir)])
+
+
+def test_websockets(tmpdir):
+    check_call(["git", "clone", "https://github.com/aaugustin/websockets.git", str(tmpdir)])
+    main(
+        [
+            "--check-only",
+            "--diff",
+            str(tmpdir),
+            "--skip",
+            "example",
+            "--skip",
+            "docs",
+            "--skip",
+            "compliance",
+        ]
+    )
+
+
+def test_airflow(tmpdir):
+    check_call(["git", "clone", "https://github.com/apache/airflow.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir)])
+
+
+def test_typeshed(tmpdir):
+    check_call(["git", "clone", "https://github.com/python/typeshed.git", str(tmpdir)])
+    main(
+        [
+            "--check-only",
+            "--diff",
+            str(tmpdir),
+            "--skip",
+            "tests",
+            "--skip",
+            "scripts",
+            "--skip",
+            f"{tmpdir}/third_party/2and3/yaml/__init__.pyi",
+        ]
+    )
+
+
+def test_pylint(tmpdir):
+    check_call(["git", "clone", "https://github.com/PyCQA/pylint.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir)])
+
+
+def test_poetry(tmpdir):
+    check_call(["git", "clone", "https://github.com/python-poetry/poetry.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir), "--skip", "tests"])
+
+
+def test_hypothesis(tmpdir):
+    check_call(["git", "clone", "https://github.com/HypothesisWorks/hypothesis.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir), "--skip", "tests"])
+
+
+def test_pillow(tmpdir):
+    check_call(["git", "clone", "https://github.com/python-pillow/Pillow.git", str(tmpdir)])
+    main(["--check-only", "--diff", str(tmpdir), "--skip", "tests"])
diff --git a/tests/test_integration.py b/tests/test_integration.py
deleted file mode 100644
index 9f10c221..00000000
--- a/tests/test_integration.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from subprocess import check_call
-
-from isort.main import main
-
-
-def test_django(tmpdir):
-    check_call(["git", "clone", "https://github.com/django/django", str(tmpdir)])
-    isort_target_dirs = [
-        str(target_dir) for target_dir in (tmpdir / "django", tmpdir / "tests", tmpdir / "scripts")
-    ]
-    main(["--check-only", "--diff", *isort_target_dirs])
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/unit/__init__.py
diff --git a/tests/conftest.py b/tests/unit/conftest.py
index 3b778a70..09e4acbd 100644
--- a/tests/conftest.py
+++ b/tests/unit/conftest.py
@@ -5,7 +5,7 @@ from pathlib import Path
 import pytest
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-SRC_DIR = os.path.abspath(os.path.join(TEST_DIR, "../isort/"))
+SRC_DIR = os.path.abspath(os.path.join(TEST_DIR, "../../isort/"))
 
 
 @pytest.fixture
diff --git a/tests/example_crlf_file.py b/tests/unit/example_crlf_file.py
index 78db9518..78db9518 100644
--- a/tests/example_crlf_file.py
+++ b/tests/unit/example_crlf_file.py
diff --git a/tests/test_action_comments.py b/tests/unit/test_action_comments.py
index 508db0d2..508db0d2 100644
--- a/tests/test_action_comments.py
+++ b/tests/unit/test_action_comments.py
diff --git a/tests/test_api.py b/tests/unit/test_api.py
index 1b3ed370..1b3ed370 100644
--- a/tests/test_api.py
+++ b/tests/unit/test_api.py
diff --git a/tests/test_comments.py b/tests/unit/test_comments.py
index 4098f8ec..4098f8ec 100644
--- a/tests/test_comments.py
+++ b/tests/unit/test_comments.py
diff --git a/tests/test_deprecated_finders.py b/tests/unit/test_deprecated_finders.py
index bbd43596..bbd43596 100644
--- a/tests/test_deprecated_finders.py
+++ b/tests/unit/test_deprecated_finders.py
diff --git a/tests/test_exceptions.py b/tests/unit/test_exceptions.py
index 8a6f60fc..8a6f60fc 100644
--- a/tests/test_exceptions.py
+++ b/tests/unit/test_exceptions.py
diff --git a/tests/test_format.py b/tests/unit/test_format.py
index 3a8f7236..3a8f7236 100644
--- a/tests/test_format.py
+++ b/tests/unit/test_format.py
diff --git a/tests/test_hooks.py b/tests/unit/test_hooks.py
index 083fbfd4..083fbfd4 100644
--- a/tests/test_hooks.py
+++ b/tests/unit/test_hooks.py
diff --git a/tests/test_importable.py b/tests/unit/test_importable.py
index c48ce333..d45ca506 100644
--- a/tests/test_importable.py
+++ b/tests/unit/test_importable.py
@@ -4,6 +4,7 @@ import pytest
 
 def test_importable():
     """Simple smoketest to ensure all isort modules are importable"""
+
     import isort
     import isort._future
     import isort._future._dataclasses
diff --git a/tests/test_io.py b/tests/unit/test_io.py
index a4479c86..a4479c86 100644
--- a/tests/test_io.py
+++ b/tests/unit/test_io.py
diff --git a/tests/test_isort.py b/tests/unit/test_isort.py
index bcbc0f3a..bcbc0f3a 100644
--- a/tests/test_isort.py
+++ b/tests/unit/test_isort.py
diff --git a/tests/test_literal.py b/tests/unit/test_literal.py
index 0dd7458c..0dd7458c 100644
--- a/tests/test_literal.py
+++ b/tests/unit/test_literal.py
diff --git a/tests/test_main.py b/tests/unit/test_main.py
index 0d273871..0d273871 100644
--- a/tests/test_main.py
+++ b/tests/unit/test_main.py
diff --git a/tests/test_output.py b/tests/unit/test_output.py
index 2457f70d..2457f70d 100644
--- a/tests/test_output.py
+++ b/tests/unit/test_output.py
diff --git a/tests/test_parse.py b/tests/unit/test_parse.py
index 685ebcf7..685ebcf7 100644
--- a/tests/test_parse.py
+++ b/tests/unit/test_parse.py
diff --git a/tests/test_place.py b/tests/unit/test_place.py
index cc231d5f..cc231d5f 100644
--- a/tests/test_place.py
+++ b/tests/unit/test_place.py
diff --git a/tests/test_pylama_isort.py b/tests/unit/test_pylama_isort.py
index b7b78c13..b7b78c13 100644
--- a/tests/test_pylama_isort.py
+++ b/tests/unit/test_pylama_isort.py
diff --git a/tests/test_regressions.py b/tests/unit/test_regressions.py
index 4d5758c1..4d5758c1 100644
--- a/tests/test_regressions.py
+++ b/tests/unit/test_regressions.py
diff --git a/tests/test_settings.py b/tests/unit/test_settings.py
index fb231618..fb231618 100644
--- a/tests/test_settings.py
+++ b/tests/unit/test_settings.py
diff --git a/tests/test_setuptools_command.py b/tests/unit/test_setuptools_command.py
index c8da0d23..c8da0d23 100644
--- a/tests/test_setuptools_command.py
+++ b/tests/unit/test_setuptools_command.py
diff --git a/tests/test_ticketed_features.py b/tests/unit/test_ticketed_features.py
index 5828ec9d..5828ec9d 100644
--- a/tests/test_ticketed_features.py
+++ b/tests/unit/test_ticketed_features.py
diff --git a/tests/test_wrap.py b/tests/unit/test_wrap.py
index 2b8ec6fa..2b8ec6fa 100644
--- a/tests/test_wrap.py
+++ b/tests/unit/test_wrap.py
diff --git a/tests/test_wrap_modes.py b/tests/unit/test_wrap_modes.py
index d27f92e7..d27f92e7 100644
--- a/tests/test_wrap_modes.py
+++ b/tests/unit/test_wrap_modes.py