path: root/Tools/Scripts/webkitpy/layout_tests/port/driver.py

# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the Google name nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import base64
import copy
import logging
import re
import shlex
import sys
import time

from webkitpy.common.system import path


_log = logging.getLogger(__name__)


class DriverInput(object):
    def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, args=None):
        self.test_name = test_name
        self.timeout = timeout  # in ms
        self.image_hash = image_hash
        self.should_run_pixel_test = should_run_pixel_test
        self.args = args or []


class DriverOutput(object):
    """Groups information about a output from driver for easy passing
    and post-processing of data."""

    strip_patterns = []
    strip_patterns.append((re.compile('at \(-?[0-9]+,-?[0-9]+\) *'), ''))
    strip_patterns.append((re.compile('size -?[0-9]+x-?[0-9]+ *'), ''))
    strip_patterns.append((re.compile('text run width -?[0-9]+: '), ''))
    strip_patterns.append((re.compile('text run width -?[0-9]+ [a-zA-Z ]+: '), ''))
    strip_patterns.append((re.compile('RenderButton {BUTTON} .*'), 'RenderButton {BUTTON}'))
    strip_patterns.append((re.compile('RenderImage {INPUT} .*'), 'RenderImage {INPUT}'))
    strip_patterns.append((re.compile('RenderBlock {INPUT} .*'), 'RenderBlock {INPUT}'))
    strip_patterns.append((re.compile('RenderTextControl {INPUT} .*'), 'RenderTextControl {INPUT}'))
    strip_patterns.append((re.compile('\([0-9]+px'), 'px'))
    strip_patterns.append((re.compile(' *" *\n +" *'), ' '))
    strip_patterns.append((re.compile('" +$'), '"'))
    strip_patterns.append((re.compile('- '), '-'))
    strip_patterns.append((re.compile('\s+"\n'), '"\n'))
    strip_patterns.append((re.compile('scrollWidth [0-9]+'), 'scrollWidth'))
    strip_patterns.append((re.compile('scrollHeight [0-9]+'), 'scrollHeight'))

    def __init__(self, text, image, image_hash, audio, crash=False,
            test_time=0, timeout=False, error='', crashed_process_name='??',
            crashed_pid=None, crash_log=None):
        # FIXME: Args could be renamed to better clarify what they do.
        self.text = text
        self.image = image  # May be empty-string if the test crashes.
        self.image_hash = image_hash
        self.image_diff = None  # image_diff gets filled in after construction.
        self.audio = audio  # Binary format is port-dependent.
        self.crash = crash
        self.crashed_process_name = crashed_process_name
        self.crashed_pid = crashed_pid
        self.crash_log = crash_log
        self.test_time = test_time
        self.timeout = timeout
        self.error = error  # stderr output

    def has_stderr(self):
        return bool(self.error)

    def strip_metrics(self):
        if not self.text:
            return
        for pattern in self.strip_patterns:
            self.text = re.sub(pattern[0], pattern[1], self.text)


class Driver(object):
    """object for running test(s) using DumpRenderTree/WebKitTestRunner."""

    def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
        """Initialize a Driver to subsequently run tests.

        Typically this routine will spawn DumpRenderTree in a config
        ready for subsequent input.

        port - reference back to the port object.
        worker_number - identifier for a particular worker/driver instance
        """
        self._port = port
        self._worker_number = worker_number
        self._no_timeout = no_timeout

        self._driver_tempdir = None
        # WebKitTestRunner can report back subprocess crashes by printing
        # "#CRASHED - PROCESSNAME".  Since those can happen at any time
        # and ServerProcess won't be aware of them (since the actual tool
        # didn't crash, just a subprocess) we record the crashed subprocess name here.
        self._crashed_process_name = None
        self._crashed_pid = None

        # WebKitTestRunner can report back subprocesses that became unresponsive
        # This could mean they crashed.
        self._subprocess_was_unresponsive = False

        # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
        # stderr output, as well as if we've seen #EOF on this driver instance.
        # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
        # instead scope these locally in run_test.
        self.error_from_test = str()
        self.err_seen_eof = False
        self._server_process = None

    def __del__(self):
        self.stop()

    def run_test(self, driver_input):
        """Run a single test and return the results.

        Note that it is okay if a test times out or crashes and leaves
        the driver in an indeterminate state. The upper layers of the program
        are responsible for cleaning up and ensuring things are okay.

        Returns a DriverOuput object.
        """
        start_time = time.time()
        self.start(driver_input.should_run_pixel_test, driver_input.args)
        test_begin_time = time.time()
        self.error_from_test = str()
        self.err_seen_eof = False

        command = self._command_from_driver_input(driver_input)
        deadline = test_begin_time + int(driver_input.timeout) / 1000.0

        self._server_process.write(command)
        text, audio = self._read_first_block(deadline)  # First block is either text or audio
        image, actual_image_hash = self._read_optional_image_block(deadline)  # The second (optional) block is image data.

        # We may not have read all of the output if an error (crash) occured.
        # Since some platforms output the stacktrace over error, we should
        # dump any buffered error into self.error_from_test.
        # FIXME: We may need to also read stderr until the process dies?
        self.error_from_test += self._server_process.pop_all_buffered_stderr()

        crash_log = None
        if self.has_crashed():
            self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)

            # If we don't find a crash log use a placeholder error message instead.
            if not crash_log:
                crash_log = 'no crash log found for %s:%d.' % (self._crashed_process_name, self._crashed_pid)
                # If we were unresponsive append a message informing there may not have been a crash.
                if self._subprocess_was_unresponsive:
                    crash_log += '  Process failed to become responsive before timing out.'

        timeout = self._server_process.timed_out
        if timeout:
            # DRT doesn't have a built in timer to abort the test, so we might as well
            # kill the process directly and not wait for it to shut down cleanly (since it may not).
            self._server_process.kill()

        return DriverOutput(text, image, actual_image_hash, audio,
            crash=self.has_crashed(), test_time=time.time() - test_begin_time,
            timeout=timeout, error=self.error_from_test,
            crashed_process_name=self._crashed_process_name,
            crashed_pid=self._crashed_pid, crash_log=crash_log)

    def _get_crash_log(self, stdout, stderr, newer_than):
        return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)

    # FIXME: Seems this could just be inlined into callers.
    @classmethod
    def _command_wrapper(cls, wrapper_option):
        # Hook for injecting valgrind or other runtime instrumentation,
        # used by e.g. tools/valgrind/valgrind_tests.py.
        return shlex.split(wrapper_option) if wrapper_option else []

    HTTP_DIR = "http/tests/"
    HTTP_LOCAL_DIR = "http/tests/local/"

    def is_http_test(self, test_name):
        return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR)

    def test_to_uri(self, test_name):
        """Convert a test name to a URI."""
        if not self.is_http_test(test_name):
            return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name))

        relative_path = test_name[len(self.HTTP_DIR):]

        # TODO(dpranke): remove the SSL reference?
        if relative_path.startswith("ssl/"):
            return "https://127.0.0.1:8443/" + relative_path
        return "http://127.0.0.1:8000/" + relative_path

    def uri_to_test(self, uri):
        """Return the base layout test name for a given URI.

        This returns the test name for a given URI, e.g., if you passed in
        "file:///src/LayoutTests/fast/html/keygen.html" it would return
        "fast/html/keygen.html".

        """
        if uri.startswith("file:///"):
            prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir())
            if not prefix.endswith('/'):
                prefix += '/'
            return uri[len(prefix):]
        if uri.startswith("http://"):
            return uri.replace('http://127.0.0.1:8000/', self.HTTP_DIR)
        if uri.startswith("https://"):
            return uri.replace('https://127.0.0.1:8443/', self.HTTP_DIR)
        raise NotImplementedError('unknown url type: %s' % uri)

    def has_crashed(self):
        if self._server_process is None:
            return False
        if self._crashed_process_name:
            return True
        if self._server_process.has_crashed():
            self._crashed_process_name = self._server_process.name()
            self._crashed_pid = self._server_process.pid()
            return True
        return False

    def start(self, pixel_tests, per_test_args):
        # FIXME: Callers shouldn't normally call this, since this routine
        # may not be specifying the correct combination of pixel test and
        # per_test args.
        #
        # The only reason we have this routine at all is so the perftestrunner
        # can pause before running a test; it might be better to push that
        # into run_test() directly.
        if not self._server_process:
            self._start(pixel_tests, per_test_args)

    def _start(self, pixel_tests, per_test_args):
        self.stop()
        self._driver_tempdir = self._port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name())
        server_name = self._port.driver_name()
        environment = self._port.setup_environ_for_server(server_name)
        environment['DYLD_LIBRARY_PATH'] = self._port._build_path()
        environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
        # FIXME: We're assuming that WebKitTestRunner checks this DumpRenderTree-named environment variable.
        environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
        environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir()
        self._crashed_process_name = None
        self._crashed_pid = None
        self._server_process = self._port._server_process_constructor(self._port, server_name, self.cmd_line(pixel_tests, per_test_args), environment)
        self._server_process.start()

    def stop(self):
        if self._server_process:
            self._server_process.stop()
            self._server_process = None

        if self._driver_tempdir:
            self._port._filesystem.rmtree(str(self._driver_tempdir))
            self._driver_tempdir = None

    def cmd_line(self, pixel_tests, per_test_args):
        cmd = self._command_wrapper(self._port.get_option('wrapper'))
        cmd.append(self._port._path_to_driver())
        if self._port.get_option('gc_between_tests'):
            cmd.append('--gc-between-tests')
        if self._port.get_option('complex_text'):
            cmd.append('--complex-text')
        if self._port.get_option('threaded'):
            cmd.append('--threaded')
        if self._no_timeout:
            cmd.append('--no-timeout')
        # FIXME: We need to pass --timeout=SECONDS to WebKitTestRunner for WebKit2.

        cmd.extend(self._port.get_option('additional_drt_flag', []))
        cmd.extend(self._port.additional_drt_flag())

        cmd.extend(per_test_args)

        cmd.append('-')
        return cmd

    def _check_for_driver_crash(self, error_line):
        if error_line == "#CRASHED\n":
            # This is used on Windows to report that the process has crashed
            # See http://trac.webkit.org/changeset/65537.
            self._crashed_process_name = self._server_process.name()
            self._crashed_pid = self._server_process.pid()
        elif (error_line.startswith("#CRASHED - WebProcess")
            or error_line.startswith("#PROCESS UNRESPONSIVE - WebProcess")):
            # WebKitTestRunner uses this to report that the WebProcess subprocess crashed.
            pid = None
            m = re.search('pid (\d+)', error_line)
            if m:
                pid = int(m.group(1))
            self._crashed_process_name = 'WebProcess'
            self._crashed_pid = pid
            # FIXME: delete this after we're sure this code is working :)
            _log.debug('WebProcess crash, pid = %s, error_line = %s' % (str(pid), error_line))
            if error_line.startswith("#PROCESS UNRESPONSIVE - WebProcess"):
                self._subprocess_was_unresponsive = True
                # We want to show this since it's not a regular crash and probably we don't have a crash log.
                self.error_from_test += error_line
            return True
        return self.has_crashed()

    def _command_from_driver_input(self, driver_input):
        # FIXME: performance tests pass in full URLs instead of test names.
        if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
            command = driver_input.test_name
        elif self.is_http_test(driver_input.test_name):
            command = self.test_to_uri(driver_input.test_name)
        else:
            command = self._port.abspath_for_test(driver_input.test_name)
            if sys.platform == 'cygwin':
                command = path.cygpath(command)

        assert not driver_input.image_hash or driver_input.should_run_pixel_test

        # ' is the separator between arguments.
        if driver_input.should_run_pixel_test:
            command += "'--pixel-test"
        if driver_input.image_hash:
            command += "'" + driver_input.image_hash
        return command + "\n"

    def _read_first_block(self, deadline):
        # returns (text_content, audio_content)
        block = self._read_block(deadline)
        if block.content_type == 'audio/wav':
            return (None, block.decoded_content)
        return (block.decoded_content, None)

    def _read_optional_image_block(self, deadline):
        # returns (image, actual_image_hash)
        block = self._read_block(deadline, wait_for_stderr_eof=True)
        if block.content and block.content_type == 'image/png':
            return (block.decoded_content, block.content_hash)
        return (None, block.content_hash)

    def _read_header(self, block, line, header_text, header_attr, header_filter=None):
        if line.startswith(header_text) and getattr(block, header_attr) is None:
            value = line.split()[1]
            if header_filter:
                value = header_filter(value)
            setattr(block, header_attr, value)
            return True
        return False

    def _process_stdout_line(self, block, line):
        if (self._read_header(block, line, 'Content-Type: ', 'content_type')
            or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding')
            or self._read_header(block, line, 'Content-Length: ', '_content_length', int)
            or self._read_header(block, line, 'ActualHash: ', 'content_hash')):
            return
        # Note, we're not reading ExpectedHash: here, but we could.
        # If the line wasn't a header, we just append it to the content.
        block.content += line

    def _strip_eof(self, line):
        if line and line.endswith("#EOF\n"):
            return line[:-5], True
        return line, False

    def _read_block(self, deadline, wait_for_stderr_eof=False):
        block = ContentBlock()
        out_seen_eof = False

        while not self.has_crashed():
            if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof):
                break

            if self.err_seen_eof:
                out_line = self._server_process.read_stdout_line(deadline)
                err_line = None
            elif out_seen_eof:
                out_line = None
                err_line = self._server_process.read_stderr_line(deadline)
            else:
                out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline)

            if self._server_process.timed_out or self.has_crashed():
                break

            if out_line:
                assert not out_seen_eof
                out_line, out_seen_eof = self._strip_eof(out_line)
            if err_line:
                assert not self.err_seen_eof
                err_line, self.err_seen_eof = self._strip_eof(err_line)

            if out_line:
                if out_line[-1] != "\n":
                    _log.error("Last character read from DRT stdout line was not a newline!  This indicates either a NRWT or DRT bug.")
                content_length_before_header_check = block._content_length
                self._process_stdout_line(block, out_line)
                # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header.
                # Don't wait until we're done with headers, just read the binary blob right now.
                if content_length_before_header_check != block._content_length:
                    block.content = self._server_process.read_stdout(deadline, block._content_length)

            if err_line:
                if self._check_for_driver_crash(err_line):
                    break
                self.error_from_test += err_line

        block.decode_content()
        return block


class ContentBlock(object):
    def __init__(self):
        self.content_type = None
        self.encoding = None
        self.content_hash = None
        self._content_length = None
        # Content is treated as binary data even though the text output is usually UTF-8.
        self.content = str()  # FIXME: Should be bytearray() once we require Python 2.6.
        self.decoded_content = None

    def decode_content(self):
        if self.encoding == 'base64' and self.content is not None:
            self.decoded_content = base64.b64decode(self.content)
        else:
            self.decoded_content = self.content

class DriverProxy(object):
    """A wrapper for managing two Driver instances, one with pixel tests and
    one without. This allows us to handle plain text tests and ref tests with a
    single driver."""

    def __init__(self, port, worker_number, driver_instance_constructor, pixel_tests, no_timeout):
        self._port = port
        self._worker_number = worker_number
        self._driver_instance_constructor = driver_instance_constructor
        self._no_timeout = no_timeout

        # FIXME: We shouldn't need to create a driver until we actually run a test.
        self._driver = self._make_driver(pixel_tests)
        self._running_drivers = {}
        self._running_drivers[self._cmd_line_as_key(pixel_tests, [])] = self._driver

    def _make_driver(self, pixel_tests):
        return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout)

    # FIXME: this should be a @classmethod (or implemented on Port instead).
    def is_http_test(self, test_name):
        return self._driver.is_http_test(test_name)

    # FIXME: this should be a @classmethod (or implemented on Port instead).
    def test_to_uri(self, test_name):
        return self._driver.test_to_uri(test_name)

    # FIXME: this should be a @classmethod (or implemented on Port instead).
    def uri_to_test(self, uri):
        return self._driver.uri_to_test(uri)

    def run_test(self, driver_input):
        base = self._port.lookup_virtual_test_base(driver_input.test_name)
        if base:
            virtual_driver_input = copy.copy(driver_input)
            virtual_driver_input.test_name = base
            virtual_driver_input.args = self._port.lookup_virtual_test_args(driver_input.test_name)
            return self.run_test(virtual_driver_input)

        pixel_tests_needed = driver_input.should_run_pixel_test
        cmd_line_key = self._cmd_line_as_key(pixel_tests_needed, driver_input.args)
        if not cmd_line_key in self._running_drivers:
            self._running_drivers[cmd_line_key] = self._make_driver(pixel_tests_needed)

        return self._running_drivers[cmd_line_key].run_test(driver_input)

    def start(self):
        # FIXME: Callers shouldn't normally call this, since this routine
        # may not be specifying the correct combination of pixel test and
        # per_test args.
        #
        # The only reason we have this routine at all is so the perftestrunner
        # can pause before running a test; it might be better to push that
        # into run_test() directly.
        self._driver.start(self._port.get_option('pixel_tests'), [])

    def has_crashed(self):
        return any(driver.has_crashed() for driver in self._running_drivers.values())

    def stop(self):
        for driver in self._running_drivers.values():
            driver.stop()

    # FIXME: this should be a @classmethod (or implemented on Port instead).
    def cmd_line(self, pixel_tests=None, per_test_args=None):
        return self._driver.cmd_line(pixel_tests, per_test_args or [])

    def _cmd_line_as_key(self, pixel_tests, per_test_args):
        return ' '.join(self.cmd_line(pixel_tests, per_test_args))