Fixed #24522 -- Added a --shuffle option to DiscoverRunner.

3 files changed, 267 insertions, 1 deletions

diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py
index 4f7aed059f..2fc4df3133 100644
--- a/tests/test_runner/test_discover_runner.py
+++ b/tests/test_runner/test_discover_runner.py
@@ -49,6 +49,16 @@ class DiscoverRunnerTests(SimpleTestCase):
         runner = DiscoverRunner()
         self.assertFalse(runner.debug_mode)
 
+    def test_add_arguments_shuffle(self):
+        parser = ArgumentParser()
+        DiscoverRunner.add_arguments(parser)
+        ns = parser.parse_args([])
+        self.assertIs(ns.shuffle, False)
+        ns = parser.parse_args(['--shuffle'])
+        self.assertIsNone(ns.shuffle)
+        ns = parser.parse_args(['--shuffle', '5'])
+        self.assertEqual(ns.shuffle, 5)
+
     def test_add_arguments_debug_mode(self):
         parser = ArgumentParser()
         DiscoverRunner.add_arguments(parser)
@@ -58,6 +68,30 @@ class DiscoverRunnerTests(SimpleTestCase):
         ns = parser.parse_args(["--debug-mode"])
         self.assertTrue(ns.debug_mode)
 
+    def test_setup_shuffler_no_shuffle_argument(self):
+        runner = DiscoverRunner()
+        self.assertIs(runner.shuffle, False)
+        runner.setup_shuffler()
+        self.assertIsNone(runner.shuffle_seed)
+
+    def test_setup_shuffler_shuffle_none(self):
+        runner = DiscoverRunner(shuffle=None)
+        self.assertIsNone(runner.shuffle)
+        with mock.patch('random.randint', return_value=1):
+            with captured_stdout() as stdout:
+                runner.setup_shuffler()
+        self.assertEqual(stdout.getvalue(), 'Using shuffle seed: 1 (generated)\n')
+        self.assertEqual(runner.shuffle_seed, 1)
+
+    def test_setup_shuffler_shuffle_int(self):
+        runner = DiscoverRunner(shuffle=2)
+        self.assertEqual(runner.shuffle, 2)
+        with captured_stdout() as stdout:
+            runner.setup_shuffler()
+        expected_out = 'Using shuffle seed: 2 (given)\n'
+        self.assertEqual(stdout.getvalue(), expected_out)
+        self.assertEqual(runner.shuffle_seed, 2)
+
     def test_load_tests_for_label_file_path(self):
         with change_cwd('.'):
             msg = (
@@ -266,6 +300,25 @@ class DiscoverRunnerTests(SimpleTestCase):
         self.assertIsInstance(tests[0], unittest.loader._FailedTest)
         self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest)
 
+    def test_build_suite_shuffling(self):
+        # These will result in unittest.loader._FailedTest instances rather
+        # than TestCase objects, but they are sufficient for testing.
+        labels = ['label1', 'label2', 'label3', 'label4']
+        cases = [
+            ({}, ['label1', 'label2', 'label3', 'label4']),
+            ({'reverse': True}, ['label4', 'label3', 'label2', 'label1']),
+            ({'shuffle': 8}, ['label4', 'label1', 'label3', 'label2']),
+            ({'shuffle': 8, 'reverse': True}, ['label2', 'label3', 'label1', 'label4']),
+        ]
+        for kwargs, expected in cases:
+            with self.subTest(kwargs=kwargs):
+                # Prevent writing the seed to stdout.
+                runner = DiscoverRunner(**kwargs, verbosity=0)
+                tests = runner.build_suite(test_labels=labels)
+                # The ids have the form "unittest.loader._FailedTest.label1".
+                names = [test.id().split('.')[-1] for test in tests]
+                self.assertEqual(names, expected)
+
     def test_overridable_get_test_runner_kwargs(self):
         self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)
 
@@ -374,6 +427,52 @@ class DiscoverRunnerTests(SimpleTestCase):
         self.assertIn('Write to stderr.', stderr.getvalue())
         self.assertIn('Write to stdout.', stdout.getvalue())
 
+    def run_suite_with_runner(self, runner_class, **kwargs):
+        class MyRunner(DiscoverRunner):
+            def test_runner(self, *args, **kwargs):
+                return runner_class()
+
+        runner = MyRunner(**kwargs)
+        # Suppress logging "Using shuffle seed" to the console.
+        with captured_stdout():
+            runner.setup_shuffler()
+        with captured_stdout() as stdout:
+            try:
+                result = runner.run_suite(None)
+            except RuntimeError as exc:
+                result = str(exc)
+        output = stdout.getvalue()
+        return result, output
+
+    def test_run_suite_logs_seed(self):
+        class TestRunner:
+            def run(self, suite):
+                return '<fake-result>'
+
+        expected_prefix = 'Used shuffle seed'
+        # Test with and without shuffling enabled.
+        result, output = self.run_suite_with_runner(TestRunner)
+        self.assertEqual(result, '<fake-result>')
+        self.assertNotIn(expected_prefix, output)
+
+        result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
+        self.assertEqual(result, '<fake-result>')
+        expected_output = f'{expected_prefix}: 2 (given)\n'
+        self.assertEqual(output, expected_output)
+
+    def test_run_suite_logs_seed_exception(self):
+        """
+        run_suite() logs the seed when TestRunner.run() raises an exception.
+        """
+        class TestRunner:
+            def run(self, suite):
+                raise RuntimeError('my exception')
+
+        result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
+        self.assertEqual(result, 'my exception')
+        expected_output = 'Used shuffle seed: 2 (given)\n'
+        self.assertEqual(output, expected_output)
+
     @mock.patch('faulthandler.enable')
     def test_faulthandler_enabled(self, mocked_enable):
         with mock.patch('faulthandler.is_enabled', return_value=False):
diff --git a/tests/test_runner/test_shuffler.py b/tests/test_runner/test_shuffler.py
new file mode 100644
index 0000000000..f177249706
--- /dev/null
+++ b/tests/test_runner/test_shuffler.py
@@ -0,0 +1,102 @@
+from unittest import mock
+
+from django.test import SimpleTestCase
+from django.test.runner import Shuffler
+
+
+class ShufflerTests(SimpleTestCase):
+
+    def test_hash_text(self):
+        actual = Shuffler._hash_text('abcd')
+        self.assertEqual(actual, 'e2fc714c4727ee9395f324cd2e7f331f')
+
+    def test_hash_text_hash_algorithm(self):
+        class MyShuffler(Shuffler):
+            hash_algorithm = 'sha1'
+
+        actual = MyShuffler._hash_text('abcd')
+        self.assertEqual(actual, '81fe8bfe87576c3ecb22426f8e57847382917acf')
+
+    def test_init(self):
+        shuffler = Shuffler(100)
+        self.assertEqual(shuffler.seed, 100)
+        self.assertEqual(shuffler.seed_source, 'given')
+
+    def test_init_none_seed(self):
+        with mock.patch('random.randint', return_value=200):
+            shuffler = Shuffler(None)
+        self.assertEqual(shuffler.seed, 200)
+        self.assertEqual(shuffler.seed_source, 'generated')
+
+    def test_init_no_seed_argument(self):
+        with mock.patch('random.randint', return_value=300):
+            shuffler = Shuffler()
+        self.assertEqual(shuffler.seed, 300)
+        self.assertEqual(shuffler.seed_source, 'generated')
+
+    def test_seed_display(self):
+        shuffler = Shuffler(100)
+        shuffler.seed_source = 'test'
+        self.assertEqual(shuffler.seed_display, '100 (test)')
+
+    def test_hash_item_seed(self):
+        cases = [
+            (1234, '64ad3fb166ddb41a2ca24f1803b8b722'),
+            # Passing a string gives the same value.
+            ('1234', '64ad3fb166ddb41a2ca24f1803b8b722'),
+            (5678, '4dde450ad339b6ce45a0a2666e35b975'),
+        ]
+        for seed, expected in cases:
+            with self.subTest(seed=seed):
+                shuffler = Shuffler(seed=seed)
+                actual = shuffler._hash_item('abc', lambda x: x)
+                self.assertEqual(actual, expected)
+
+    def test_hash_item_key(self):
+        cases = [
+            (lambda x: x, '64ad3fb166ddb41a2ca24f1803b8b722'),
+            (lambda x: x.upper(), 'ee22e8597bff91742affe4befbf4649a'),
+        ]
+        for key, expected in cases:
+            with self.subTest(key=key):
+                shuffler = Shuffler(seed=1234)
+                actual = shuffler._hash_item('abc', key)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_key(self):
+        cases = [
+            (lambda x: x, ['a', 'd', 'b', 'c']),
+            (lambda x: x.upper(), ['d', 'c', 'a', 'b']),
+        ]
+        for num, (key, expected) in enumerate(cases, start=1):
+            with self.subTest(num=num):
+                shuffler = Shuffler(seed=1234)
+                actual = shuffler.shuffle(['a', 'b', 'c', 'd'], key)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_consistency(self):
+        seq = [str(n) for n in range(5)]
+        cases = [
+            (None, ['3', '0', '2', '4', '1']),
+            (0, ['3', '2', '4', '1']),
+            (1, ['3', '0', '2', '4']),
+            (2, ['3', '0', '4', '1']),
+            (3, ['0', '2', '4', '1']),
+            (4, ['3', '0', '2', '1']),
+        ]
+        shuffler = Shuffler(seed=1234)
+        for index, expected in cases:
+            with self.subTest(index=index):
+                if index is None:
+                    new_seq = seq
+                else:
+                    new_seq = seq.copy()
+                    del new_seq[index]
+                actual = shuffler.shuffle(new_seq, lambda x: x)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_same_hash(self):
+        shuffler = Shuffler(seed=1234)
+        msg = "item 'A' has same hash 'a56ce89262959e151ee2266552f1819c' as item 'a'"
+        with self.assertRaisesMessage(RuntimeError, msg):
+            shuffler.shuffle(['a', 'b', 'A'], lambda x: x.upper())
diff --git a/tests/test_runner/tests.py b/tests/test_runner/tests.py
index 6348569dfc..77a756a1de 100644
--- a/tests/test_runner/tests.py
+++ b/tests/test_runner/tests.py
@@ -1,6 +1,7 @@
 """
 Tests for django test runner
 """
+import collections.abc
 import unittest
 from unittest import mock
 
@@ -14,7 +15,9 @@ from django.core.management.base import SystemCheckError
 from django.test import (
     SimpleTestCase, TransactionTestCase, skipUnlessDBFeature,
 )
-from django.test.runner import DiscoverRunner, reorder_tests
+from django.test.runner import (
+    DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests,
+)
 from django.test.testcases import connections_support_transactions
 from django.test.utils import (
     captured_stderr, dependency_ordered, get_unique_databases_and_mirrors,
@@ -126,6 +129,68 @@ class TestSuiteTests(SimpleTestCase):
         self.assertEqual(len(tests), 4)
         self.assertNotIsInstance(tests[0], unittest.TestSuite)
 
+    def make_tests(self):
+        """Return an iterable of tests."""
+        suite = self.make_test_suite()
+        tests = list(iter_test_cases(suite))
+        return tests
+
+    def test_shuffle_tests(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        shuffled_tests = shuffle_tests(tests, shuffler)
+        self.assertIsInstance(shuffled_tests, collections.abc.Iterator)
+        self.assertTestNames(shuffled_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_no_arguments(self):
+        tests = self.make_tests()
+        reordered_tests = reorder_test_bin(tests)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
+        ])
+
+    def test_reorder_test_bin_reverse(self):
+        tests = self.make_tests()
+        reordered_tests = reorder_test_bin(tests, reverse=True)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_random(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_test_bin(tests, shuffler=shuffler)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_random_and_reverse(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
+        ])
+
+    def test_reorder_tests_random(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
     def test_reorder_tests_reverse_with_duplicates(self):
         class Tests1(unittest.TestCase):
             def test1(self):