path: root/chromium/third_party/blink/tools/blinkpy/tool/commands/rebaseline_unittest.py

# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import optparse
import unittest

from blinkpy.common.net.buildbot import Build
from blinkpy.common.net.layout_test_results import LayoutTestResults
from blinkpy.common.path_finder import RELATIVE_WEB_TESTS
from blinkpy.common.system.executive_mock import MockExecutive
from blinkpy.tool.commands.rebaseline import (
    AbstractParallelRebaselineCommand, Rebaseline, TestBaselineSet
)
from blinkpy.tool.mock_tool import MockBlinkTool
from blinkpy.web_tests.builder_list import BuilderList
from blinkpy.web_tests.port.factory_mock import MockPortFactory


class BaseTestCase(unittest.TestCase):

    WEB_PREFIX = 'https://test-results.appspot.com/data/layout_results/MOCK_Mac10_11/results/layout-test-results'

    command_constructor = lambda: None

    def setUp(self):
        self.tool = MockBlinkTool()
        self.command = self.command_constructor()
        self.command._tool = self.tool   # pylint: disable=protected-access
        self.tool.builders = BuilderList({
            'MOCK Mac10.10 (dbg)': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Debug']},
            'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']},
            'MOCK Mac10.11 (dbg)': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Debug']},
            'MOCK Mac10.11 ASAN': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
            'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
            'MOCK Precise': {'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release']},
            'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
            'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
            'MOCK Win7 (dbg)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7 (dbg)(1)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7 (dbg)(2)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
            'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']},
        })
        self.mac_port = self.tool.port_factory.get_from_builder_name('MOCK Mac10.11')
        self.test_expectations_path = self.mac_port.path_to_generic_test_expectations_file()

        # This file must exist for Port classes to function properly.
        self._write('VirtualTestSuites', '[]')
        # Create some dummy tests (note _setup_mock_build_data uses the same test names).
        self._write('userscripts/first-test.html', 'Dummy test contents')
        self._write('userscripts/second-test.html', 'Dummy test contents')

        # In AbstractParallelRebaselineCommand._rebaseline_commands, a default port
        # object is gotten using self.tool.port_factory.get(), which is used to get
        # test paths -- and the layout tests directory may be different for the "test"
        # ports and real ports. Since only "test" ports are used in this class,
        # we can make the default port also a "test" port.
        self.original_port_factory_get = self.tool.port_factory.get
        test_port = self.tool.port_factory.get('test')

        def get_test_port(port_name=None, options=None, **kwargs):
            if not port_name:
                return test_port
            return self.original_port_factory_get(port_name, options, **kwargs)

        self.tool.port_factory.get = get_test_port

    def tearDown(self):
        self.tool.port_factory.get = self.original_port_factory_get

    def _expand(self, path):
        if self.tool.filesystem.isabs(path):
            return path
        return self.tool.filesystem.join(self.mac_port.layout_tests_dir(), path)

    def _read(self, path):
        return self.tool.filesystem.read_text_file(self._expand(path))

    def _write(self, path, contents):
        self.tool.filesystem.write_text_file(self._expand(path), contents)

    def _zero_out_test_expectations(self):
        for port_name in self.tool.port_factory.all_port_names():
            port = self.tool.port_factory.get(port_name)
            for path in port.expectations_files():
                self._write(path, '')
        self.tool.filesystem.written_files = {}

    def _setup_mock_build_data(self):
        for builder in ['MOCK Win7', 'MOCK Win7 (dbg)', 'MOCK Mac10.11']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'userscripts': {
                        'first-test.html': {
                            'expected': 'PASS',
                            'actual': 'IMAGE+TEXT',
                            'is_unexpected': True,
                        },
                        'second-test.html': {
                            'expected': 'FAIL',
                            'actual': 'IMAGE+TEXT',
                        }
                    }
                }
            }))



class TestAbstractParallelRebaselineCommand(BaseTestCase):
    """Tests for the base class of multiple rebaseline commands.

    This class only contains test cases for utility methods. Some common
    behaviours of various rebaseline commands are tested in TestRebaseline.
    """

    command_constructor = AbstractParallelRebaselineCommand

    def test_builders_to_fetch_from(self):
        # pylint: disable=protected-access
        builders_to_fetch = self.command._builders_to_fetch_from(
            ['MOCK Win10', 'MOCK Win7 (dbg)(1)', 'MOCK Win7 (dbg)(2)', 'MOCK Win7'])
        self.assertEqual(builders_to_fetch, {'MOCK Win7', 'MOCK Win10'})

    def test_generic_baseline_paths(self):
        test_baseline_set = TestBaselineSet(self.tool)
        # Multiple ports shouldn't produce duplicate baseline paths.
        test_baseline_set.add('passes/text.html', Build('MOCK Win7'))
        test_baseline_set.add('passes/text.html', Build('MOCK Win10'))

        # pylint: disable=protected-access
        baseline_paths = self.command._generic_baseline_paths(test_baseline_set)
        self.assertEqual(baseline_paths, [
            '/test.checkout/wtests/passes/text-expected.png',
            '/test.checkout/wtests/passes/text-expected.txt',
            '/test.checkout/wtests/passes/text-expected.wav',
        ])

    def test_unstaged_baselines(self):
        git = self.tool.git()
        git.unstaged_changes = lambda: {
            RELATIVE_WEB_TESTS + 'x/foo-expected.txt': 'M',
            RELATIVE_WEB_TESTS + 'x/foo-expected.something': '?',
            RELATIVE_WEB_TESTS + 'x/foo-expected.png': '?',
            RELATIVE_WEB_TESTS + 'x/foo.html': 'M',
            'docs/something.md': '?',
        }
        self.assertEqual(
            self.command.unstaged_baselines(),
            [
                '/mock-checkout/' + RELATIVE_WEB_TESTS + 'x/foo-expected.png',
                '/mock-checkout/' + RELATIVE_WEB_TESTS + 'x/foo-expected.txt',
            ])


class TestRebaseline(BaseTestCase):
    """Tests for the blink_tool.py rebaseline command.

    Also tests some common behaviours of all rebaseline commands.
    """

    command_constructor = Rebaseline

    def setUp(self):
        super(TestRebaseline, self).setUp()
        self.tool.executive = MockExecutive()
        self._setup_mock_build_data()

    def tearDown(self):
        super(TestRebaseline, self).tearDown()

    @staticmethod
    def options(**kwargs):
        return optparse.Values(dict({
            'optimize': True,
            'verbose': True,
            'results_directory': None
        }, **kwargs))

    def test_rebaseline_test_passes_on_all_builders(self):
        self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'first-test.html': {
                        'expected': 'REBASELINE',
                        'actual': 'PASS'
                    }
                }
            }
        }))

        self._write(self.test_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7'))
        self.command.rebaseline(self.options(), test_baseline_set)

        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_all(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7'))
        self.command.rebaseline(self.options(), test_baseline_set)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                    '--builder', 'MOCK Win7',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]],
                [[
                    'python', 'echo', 'optimize-baselines',
                    '--verbose',
                    '--suffixes', 'txt,png',
                    'userscripts/first-test.html',
                ]]
            ])

    def test_rebaseline_debug(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7 (dbg)'))

        self.command.rebaseline(self.options(), test_baseline_set)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                    '--builder', 'MOCK Win7 (dbg)',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]],
                [[
                    'python', 'echo', 'optimize-baselines',
                    '--verbose',
                    '--suffixes', 'txt,png',
                    'userscripts/first-test.html',
                ]]
            ])

    def test_no_optimize(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7'))
        self.command.rebaseline(self.options(optimize=False), test_baseline_set)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',

                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                    '--builder', 'MOCK Win7',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]]
            ])

    def test_results_directory(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7'))
        self.command.rebaseline(self.options(optimize=False, results_directory='/tmp'), test_baseline_set)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                    '--builder', 'MOCK Win7',
                    '--results-directory', '/tmp',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]],
            ])

    def test_rebaseline_with_different_port_name(self):
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7'), 'test-win-win10')
        self.command.rebaseline(self.options(), test_baseline_set)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win10',

                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win10',
                    '--builder', 'MOCK Win7',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]],
                [[
                    'python', 'echo', 'optimize-baselines',
                    '--verbose',
                    '--suffixes', 'txt,png',
                    'userscripts/first-test.html',
                ]]
            ])


class TestRebaselineUpdatesExpectationsFiles(BaseTestCase):
    """Tests for the logic related to updating the test expectations file."""

    command_constructor = Rebaseline

    def setUp(self):
        super(TestRebaselineUpdatesExpectationsFiles, self).setUp()

        def mock_run_command(*args, **kwargs):  # pylint: disable=unused-argument
            return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "port_name": "test-mac-mac10.11"}]}\n'
        self.tool.executive = MockExecutive(run_command_fn=mock_run_command)

    @staticmethod
    def options():
        return optparse.Values({
            'optimize': False,
            'verbose': True,
            'results_directory': None
        })

    # In the following test cases, we use a mock rebaseline-test-internal to
    # pretend userscripts/first-test.html can be rebaselined on Mac10.11, so
    # the corresponding expectation (if exists) should be updated.

    def test_rebaseline_updates_expectations_file(self):
        self._write(
            self.test_expectations_path,
            ('Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n'
             'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n'))
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
             'bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n'))

    def test_rebaseline_updates_expectations_file_all_platforms(self):
        self._write(self.test_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')

    def test_rebaseline_handles_platform_skips(self):
        # This test is just like test_rebaseline_updates_expectations_file_all_platforms(),
        # except that if a particular port happens to SKIP a test in an overrides file,
        # we count that as passing, and do not think that we still need to rebaseline it.
        self._write(self.test_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('NeverFixTests', 'Bug(y) [ Android ] userscripts [ WontFix ]\n')
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')

    def test_rebaseline_handles_skips_in_file(self):
        # This test is like test_rebaseline_handles_platform_skips, except that the
        # Skip is in the same (generic) file rather than a platform file. In this case,
        # the Skip line should be left unmodified. Note that the first line is now
        # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
        # the second line.
        self._write(self.test_expectations_path,
                    ('Bug(x) [ Linux Mac ] userscripts/first-test.html [ Failure ]\n'
                     'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Linux Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
             'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))

    def test_rebaseline_handles_smoke_tests(self):
        # This test is just like test_rebaseline_handles_platform_skips, except that we check for
        # a test not being in the SmokeTests file, instead of using overrides files.
        # If a test is not part of the smoke tests, we count that as passing on ports that only
        # run smoke tests, and do not think that we still need to rebaseline it.
        self._write(self.test_expectations_path, 'Bug(x) userscripts/first-test.html [ Failure ]\n')
        self._write('SmokeTests', 'fast/html/article-element.html')
        self._setup_mock_build_data()
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failure ]\n')

    # In the following test cases, the tests produce no outputs (e.g. clean
    # passing reftests, skipped tests, etc.). Hence, there are no baselines to
    # fetch (empty baseline suffixes), and rebaseline-test-internal wouldn't be
    # called. However, in some cases the expectations still need to be updated.

    def test_rebaseline_keeps_skip_expectations(self):
        # [ Skip ], [ WontFix ] expectations should always be kept.
        self._write(self.test_expectations_path,
                    ('Bug(x) [ Mac ] userscripts/skipped-test.html [ WontFix ]\n'
                     'Bug(y) [ Win ] userscripts/skipped-test.html [ Skip ]\n'))
        self._write('userscripts/skipped-test.html', 'Dummy test contents')
        self.tool.buildbot.set_results(Build('MOCK Mac10.11'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'skipped-test.html': {
                        'expected': 'WONTFIX',
                        'actual': 'SKIP',
                    }
                }
            }
        }))
        self.tool.buildbot.set_results(Build('MOCK Win7'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'skipped-test.html': {
                        'expected': 'SKIP',
                        'actual': 'SKIP',
                    }
                }
            }
        }))
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/skipped-test.html', Build('MOCK Mac10.11'))
        test_baseline_set.add('userscripts/skipped-test.html', Build('MOCK Win7'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations,
            ('Bug(x) [ Mac ] userscripts/skipped-test.html [ WontFix ]\n'
             'Bug(y) [ Win ] userscripts/skipped-test.html [ Skip ]\n'))
        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_keeps_flaky_expectations(self):
        # Flaky expectations should be kept even if the test passes.
        self._write(self.test_expectations_path, 'Bug(x) userscripts/flaky-test.html [ Pass Failure ]\n')
        self._write('userscripts/flaky-test.html', 'Dummy test contents')
        self.tool.buildbot.set_results(Build('MOCK Mac10.11'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'flaky-test.html': {
                        'expected': 'PASS FAIL',
                        'actual': 'PASS',
                    }
                }
            }
        }))
        test_baseline_set = TestBaselineSet(self.tool)
        test_baseline_set.add('userscripts/flaky-test.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(x) userscripts/flaky-test.html [ Pass Failure ]\n')
        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_test_passes_unexpectedly(self):
        # The test passes without any output. Its expectation should be updated
        # without calling rebaseline-test-internal.
        self._write(self.test_expectations_path, 'Bug(foo) userscripts/all-pass.html [ Failure ]\n')
        self._write('userscripts/all-pass.html', 'Dummy test contents')
        test_baseline_set = TestBaselineSet(self.tool)
        self.tool.buildbot.set_results(Build('MOCK Mac10.11'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'all-pass.html': {
                        'expected': 'FAIL',
                        'actual': 'PASS',
                        'is_unexpected': True
                    }
                }
            }
        }))
        test_baseline_set.add('userscripts/all-pass.html', Build('MOCK Mac10.11'))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(foo) [ Linux Mac10.10 Win ] userscripts/all-pass.html [ Failure ]\n')
        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_test_passes_unexpectedly_everywhere(self):
        # Similar to test_rebaseline_test_passes_unexpectedly, except that the
        # test passes on all ports.
        self._write(self.test_expectations_path, 'Bug(foo) userscripts/all-pass.html [ Failure ]\n')
        self._write('userscripts/all-pass.html', 'Dummy test contents')
        test_baseline_set = TestBaselineSet(self.tool)
        for builder in ['MOCK Win7', 'MOCK Win10', 'MOCK Mac10.10', 'MOCK Mac10.11', 'MOCK Precise', 'MOCK Trusty']:
            self.tool.buildbot.set_results(Build(builder), LayoutTestResults({
                'tests': {
                    'userscripts': {
                        'all-pass.html': {
                            'expected': 'FAIL',
                            'actual': 'PASS',
                            'is_unexpected': True
                        }
                    }
                }
            }))
            test_baseline_set.add('userscripts/all-pass.html', Build(builder))

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(new_expectations, '')
        self.assertEqual(self.tool.executive.calls, [])

    def test_rebaseline_test_passes_unexpectedly_but_on_another_port(self):
        # Similar to test_rebaseline_test_passes_unexpectedly, except that the
        # build was run on a different port than the port we are rebaselining
        # (possible when rebaseline-cl --fill-missing), in which case we don't
        # update the expectations.
        self._write(self.test_expectations_path, 'Bug(foo) userscripts/all-pass.html [ Failure ]\n')
        self._write('userscripts/all-pass.html', 'Dummy test contents')
        test_baseline_set = TestBaselineSet(self.tool)
        self.tool.buildbot.set_results(Build('MOCK Mac10.11'), LayoutTestResults({
            'tests': {
                'userscripts': {
                    'all-pass.html': {
                        'expected': 'FAIL',
                        'actual': 'PASS',
                        'is_unexpected': True
                    }
                }
            }
        }))
        test_baseline_set.add('userscripts/all-pass.html', Build('MOCK Mac10.11'), 'MOCK Mac10.10')

        self.command.rebaseline(self.options(), test_baseline_set)

        new_expectations = self._read(self.test_expectations_path)
        self.assertMultiLineEqual(
            new_expectations, 'Bug(foo) userscripts/all-pass.html [ Failure ]\n')
        self.assertEqual(self.tool.executive.calls, [])


class TestRebaselineExecute(BaseTestCase):
    """Tests for the main execute function of the blink_tool.py rebaseline command."""

    command_constructor = Rebaseline

    @staticmethod
    def options():
        return optparse.Values({
            'results_directory': False,
            'optimize': False,
            'builders': None,
            'suffixes': 'txt,png',
            'verbose': True
        })

    def test_rebaseline(self):
        # pylint: disable=protected-access
        self.command._builders_to_pull_from = lambda: ['MOCK Win7']
        self._setup_mock_build_data()
        self.command.execute(self.options(), ['userscripts/first-test.html'], self.tool)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [[
                    'python', 'echo', 'copy-existing-baselines-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                ]],
                [[
                    'python', 'echo', 'rebaseline-test-internal',
                    '--verbose',
                    '--test', 'userscripts/first-test.html',
                    '--suffixes', 'txt,png',
                    '--port-name', 'test-win-win7',
                    '--builder', 'MOCK Win7',
                    '--step-name', 'webkit_layout_tests (with patch)',
                ]]
            ])

    def test_rebaseline_directory(self):
        # pylint: disable=protected-access
        self.command._builders_to_pull_from = lambda: ['MOCK Win7']

        self._setup_mock_build_data()
        self.command.execute(self.options(), ['userscripts'], self.tool)

        self.assertEqual(
            self.tool.executive.calls,
            [
                [
                    [
                        'python', 'echo', 'copy-existing-baselines-internal',
                        '--verbose',
                        '--test', 'userscripts/first-test.html',
                        '--suffixes', 'txt,png',
                        '--port-name', 'test-win-win7',
                    ],
                    [
                        'python', 'echo', 'copy-existing-baselines-internal',
                        '--verbose',
                        '--test', 'userscripts/second-test.html',
                        '--suffixes', 'txt,png',
                        '--port-name', 'test-win-win7',
                    ]
                ],
                [
                    [
                        'python', 'echo', 'rebaseline-test-internal',
                        '--verbose',
                        '--test', 'userscripts/first-test.html',
                        '--suffixes', 'txt,png',
                        '--port-name', 'test-win-win7',
                        '--builder', 'MOCK Win7',
                        '--step-name', 'webkit_layout_tests (with patch)',
                    ],
                    [
                        'python', 'echo', 'rebaseline-test-internal',
                        '--verbose',
                        '--test', 'userscripts/second-test.html',
                        '--suffixes', 'txt,png',
                        '--port-name', 'test-win-win7',
                        '--builder', 'MOCK Win7',
                        '--step-name', 'webkit_layout_tests (with patch)',
                    ]
                ]
            ])


class TestBaselineSetTest(unittest.TestCase):

    def setUp(self):
        host = MockBlinkTool()
        host.port_factory = MockPortFactory(host)
        port = host.port_factory.get()
        base_dir = port.layout_tests_dir()
        host.filesystem.write_text_file(base_dir + '/a/x.html', '<html>')
        host.filesystem.write_text_file(base_dir + '/a/y.html', '<html>')
        host.filesystem.write_text_file(base_dir + '/a/z.html', '<html>')
        host.builders = BuilderList({
            'MOCK Mac10.12': {'port_name': 'test-mac-mac10.12', 'specifiers': ['Mac10.12', 'Release']},
            'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
            'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
        })
        self.host = host

    def test_add_and_iter_tests(self):
        test_baseline_set = TestBaselineSet(host=self.host)
        test_baseline_set.add('a', Build('MOCK Trusty'))
        test_baseline_set.add('a/z.html', Build('MOCK Win10'))
        self.assertEqual(
            list(test_baseline_set),
            [
                ('a/x.html', Build(builder_name='MOCK Trusty'), 'test-linux-trusty'),
                ('a/y.html', Build(builder_name='MOCK Trusty'), 'test-linux-trusty'),
                ('a/z.html', Build(builder_name='MOCK Trusty'), 'test-linux-trusty'),
                ('a/z.html', Build(builder_name='MOCK Win10'), 'test-win-win10'),
            ])
        self.assertEqual(
            test_baseline_set.all_tests(), ['a/x.html', 'a/y.html', 'a/z.html'])

    def test_str_empty(self):
        test_baseline_set = TestBaselineSet(host=self.host)
        self.assertEqual(str(test_baseline_set), '<Empty TestBaselineSet>')

    def test_str_basic(self):
        test_baseline_set = TestBaselineSet(host=self.host)
        test_baseline_set.add('a/x.html', Build('MOCK Mac10.12'))
        test_baseline_set.add('a/x.html', Build('MOCK Win10'))
        self.assertEqual(
            str(test_baseline_set),
            ('<TestBaselineSet with:\n'
             '  a/x.html: Build(builder_name=\'MOCK Mac10.12\', build_number=None), test-mac-mac10.12\n'
             '  a/x.html: Build(builder_name=\'MOCK Win10\', build_number=None), test-win-win10>'))

    def test_getters(self):
        test_baseline_set = TestBaselineSet(host=self.host)
        test_baseline_set.add('a/x.html', Build('MOCK Mac10.12'))
        test_baseline_set.add('a/x.html', Build('MOCK Win10'))
        self.assertEqual(test_baseline_set.test_prefixes(), ['a/x.html'])
        self.assertEqual(
            test_baseline_set.build_port_pairs('a/x.html'),
            [
                (Build(builder_name='MOCK Mac10.12'), 'test-mac-mac10.12'),
                (Build(builder_name='MOCK Win10'), 'test-win-win10')
            ])