path: root/flake8/tests/_test_warnings.py

"""
    _test_warnings.py

    Tests for the warnings that are emitted by flake8.

    This module is named _test_warnings instead of test_warnings so that a
    normal nosetests run does not collect it. The tests in this module pass
    when they are run alone, but they fail when they are run along with other
    tests (nosetests --with-isolation doesn't help).

    In tox.ini, these tests are run separately.

"""

from __future__ import with_statement

import os
import warnings
import unittest
try:
    from unittest import mock
except ImportError:
    import mock  # < PY33

from flake8 import engine
from flake8.util import is_windows

# The Problem
# ------------
#
# Some of the tests in this module pass when this module is run on its own, but
# they fail when this module is run as part of the whole test suite. These are
# the problematic tests:
#
#   test_jobs_verbose
#   test_stdin_jobs_warning
#
# On some platforms, the warnings.capture_warnings function doesn't work
# properly when run with the other flake8 tests. It drops some warnings, even
# though the warnings filter is set to 'always'. However, when run separately,
# these tests pass.
#
# This problem only occurs on Windows, with Python 3.3 and older. Maybe it's
# related to PEP 446 - Inheritable file descriptors?
#
#
#
#
# Things that didn't work
# ------------
#
# Nose --attr
#   I tried using the nosetests --attr feature to run the tests separately. I
#   put the following in setup.cfg
#
#   [nosetests]
#   atttr=!run_alone
#
#   Then I added a tox section thst did this
#
#   nosetests --attr=run_alone
#
#   However, the command line --attr would not override the config file --attr,
#   so the special tox section wound up runing all the tests, and failing.
#
#
#
# Nose --with-isolation
#   The nosetests --with-isolation flag did not help.
#
#
#
# unittest.skipIf
#   I tried decorating the problematic tests with the unittest.skipIf
#   decorator.
#
#    @unittest.skipIf(is_windows() and sys.version_info < (3, 4),
#                     "Fails on Windows with Python < 3.4 when run with other"
#                     " tests.")
#
#   The idea is, skip the tests in the main test run, on affected platforms.
#   Then, only on those platforms, come back in later and run the tests
#   separately.
#
#   I added a new stanza to tox.ini, to run the tests separately on the
#   affected platforms.
#
#   nosetests --no-skip
#
#   I ran in to a bug in the nosetests skip plugin. It would report the test as
#   having been run, but it would not actually run the test. So, when run with
#   --no-skip, the following test would be reported as having run and passed!
#
#   @unittest.skip("This passes o_o")
#   def test_should_fail(self):
#       assert 0
#
#   This bug has been reported here:
#   "--no-skip broken with Python 2.7"
#   https://github.com/nose-devs/nose/issues/512
#
#
#
# py.test
#
#   I tried using py.test, and its @pytest.mark.xfail decorator. I added some
#   separate stanzas in tox, and useing the pytest --runxfail option to run the
#   tests separately. This allows us to run all the tests together, on
#   platforms that allow it. On platforms that don't allow us to run the tests
#   all together, this still runs all the tests, but in two separate steps.
#
#   This is the same solution as the nosetests --no-skip solution I described
#   above, but --runxfail does not have the same bug as --no-skip.
#
#   This has the advantage that all tests are discoverable by default, outside
#   of tox. However, nose does not recognize the pytest.mark.xfail decorator.
#   So, if a user runs nosetests, it still tries to run the problematic tests
#   together with the rest of the test suite, causing them to fail.
#
#
#
#
#
#
# Solution
# ------------
# Move the problematic tests to _test_warnings.py, so nose.collector will not
# find them. Set up a separate section in tox.ini that runs this:
#
#     nosetests flake8.tests._test_warnings
#
# This allows all tests to pass on all platforms, when run through tox.
# However, it means that, even on unaffected platforms, the problematic tests
# are not discovered and run outside of tox (if the user just runs nosetests
# manually, for example).


class IntegrationTestCaseWarnings(unittest.TestCase):
    """Integration style tests to check that warnings are issued properly for
    different command line options."""

    windows_warning_text = ("The --jobs option is not available on Windows."
                            " Ignoring --jobs arguments.")
    stdin_warning_text = ("The --jobs option is not compatible with"
                          " supplying input using - . Ignoring --jobs"
                          " arguments.")

    def this_file(self):
        """Return the real path of this file."""
        this_file = os.path.realpath(__file__)
        if this_file.endswith("pyc"):
            this_file = this_file[:-1]
        return this_file

    @staticmethod
    def get_style_guide_with_warnings(engine, *args, **kwargs):
        """
        Return a style guide object (obtained by calling
        engine.get_style_guide) and a list of the warnings that were raised in
        the process.

        Note: not threadsafe
        """

        # Note
        # https://docs.python.org/2/library/warnings.html
        #
        # The catch_warnings manager works by replacing and then later
        # restoring the module's showwarning() function and internal list of
        # filter specifications. This means the context manager is modifying
        # global state and therefore is not thread-safe

        with warnings.catch_warnings(record=True) as collected_warnings:
            # Cause all warnings to always be triggered.
            warnings.simplefilter("always")

            # Get the style guide
            style_guide = engine.get_style_guide(*args, **kwargs)

        # Now that the warnings have been collected, return the style guide and
        # the warnings.
        return (style_guide, collected_warnings)

    def verify_warnings(self, collected_warnings, expected_warnings):
        """
        Verifies that collected_warnings is a sequence that contains user
        warnings that match the sequence of string values passed in as
        expected_warnings.
        """
        if expected_warnings is None:
            expected_warnings = []

        collected_user_warnings = [w for w in collected_warnings
                                   if issubclass(w.category, UserWarning)]

        self.assertEqual(len(collected_user_warnings),
                         len(expected_warnings))

        collected_warnings_set = set(str(warning.message)
                                     for warning
                                     in collected_user_warnings)
        expected_warnings_set = set(expected_warnings)
        self.assertEqual(collected_warnings_set, expected_warnings_set)

    def check_files_collect_warnings(self,
                                     arglist=[],
                                     explicit_stdin=False,
                                     count=0,
                                     verbose=False):
        """Call check_files and collect any warnings that are issued."""
        if verbose:
            arglist.append('--verbose')
        if explicit_stdin:
            target_file = "-"
        else:
            target_file = self.this_file()
        argv = ['flake8'] + arglist + [target_file]
        with mock.patch("sys.argv", argv):
            (style_guide,
             collected_warnings,
             ) = self.get_style_guide_with_warnings(engine,
                                                    parse_argv=True)
            report = style_guide.check_files()
        self.assertEqual(report.total_errors, count)
        return style_guide, report, collected_warnings

    def check_files_no_warnings_allowed(self,
                                        arglist=[],
                                        explicit_stdin=False,
                                        count=0,
                                        verbose=False):
        """Call check_files, and assert that there were no warnings issued."""
        (style_guide,
         report,
         collected_warnings,
         ) = self.check_files_collect_warnings(arglist=arglist,
                                               explicit_stdin=explicit_stdin,
                                               count=count,
                                               verbose=verbose)
        self.verify_warnings(collected_warnings, expected_warnings=None)
        return style_guide, report

    def _job_tester(self, jobs, verbose=False):
        # mock stdout.flush so we can count the number of jobs created
        with mock.patch('sys.stdout.flush') as mocked:
            (guide,
             report,
             collected_warnings,
             ) = self.check_files_collect_warnings(
                arglist=['--jobs=%s' % jobs],
                verbose=verbose)

            if is_windows():
                # The code path where guide.options.jobs gets converted to an
                # int is not run on windows. So, do the int conversion here.
                self.assertEqual(int(guide.options.jobs), jobs)
                # On windows, call count is always zero.
                self.assertEqual(mocked.call_count, 0)
            else:
                self.assertEqual(guide.options.jobs, jobs)
                self.assertEqual(mocked.call_count, jobs)

            expected_warings = []
            if verbose and is_windows():
                expected_warings.append(self.windows_warning_text)
            self.verify_warnings(collected_warnings, expected_warings)

    def test_jobs(self, verbose=False):
        self._job_tester(2, verbose=verbose)
        self._job_tester(10, verbose=verbose)

    def test_no_args_no_warnings(self, verbose=False):
        self.check_files_no_warnings_allowed(verbose=verbose)

    def test_stdin_jobs_warning(self, verbose=False):
        self.count = 0

        def fake_stdin():
            self.count += 1
            with open(self.this_file(), "r") as f:
                return f.read()

        with mock.patch("pep8.stdin_get_value", fake_stdin):
            (style_guide,
             report,
             collected_warnings,
             ) = self.check_files_collect_warnings(arglist=['--jobs=4'],
                                                   explicit_stdin=True,
                                                   verbose=verbose)
            expected_warings = []
            if verbose:
                expected_warings.append(self.stdin_warning_text)
                if is_windows():
                    expected_warings.append(self.windows_warning_text)
            self.verify_warnings(collected_warnings, expected_warings)
        self.assertEqual(self.count, 1)

    def test_jobs_verbose(self):
        self.test_jobs(verbose=True)

    def test_no_args_no_warnings_verbose(self):
        self.test_no_args_no_warnings(verbose=True)

    def test_stdin_jobs_warning_verbose(self):
        self.test_stdin_jobs_warning(verbose=True)


if __name__ == '__main__':
    unittest.main()