diff options
author | Simon Hausmann <simon.hausmann@nokia.com> | 2012-07-11 13:45:28 +0200 |
---|---|---|
committer | Simon Hausmann <simon.hausmann@nokia.com> | 2012-07-11 13:45:28 +0200 |
commit | d6a599dbc9d824a462b2b206316e102bf8136446 (patch) | |
tree | ecb257a5e55b2239d74b90fdad62fccd661cf286 /Tools/Scripts | |
parent | 3ccc3a85f09a83557b391aae380d3bf5f81a2911 (diff) | |
download | qtwebkit-d6a599dbc9d824a462b2b206316e102bf8136446.tar.gz |
Imported WebKit commit 8ff1f22783a32de82fee915abd55bd1b298f2644 (http://svn.webkit.org/repository/webkit/trunk@122325)
New snapshot that should work with the latest Qt build system changes
Diffstat (limited to 'Tools/Scripts')
79 files changed, 2161 insertions, 1158 deletions
diff --git a/Tools/Scripts/build-webkittestrunner b/Tools/Scripts/build-webkittestrunner index f03c1799a..e5b23545b 100755 --- a/Tools/Scripts/build-webkittestrunner +++ b/Tools/Scripts/build-webkittestrunner @@ -63,7 +63,7 @@ if (isAppleMacWebKit()) { $result = buildXCodeProject("WebKitTestRunner", $clean, XcodeOptions(), @ARGV); } elsif (isAppleWinWebKit()) { $result = buildVisualStudioProject("WebKitTestRunner.sln", $clean); -} elsif (isQt() || isGtk()) { +} elsif (isQt() || isGtk() || isEfl()) { # Qt and GTK+ build everything in one shot. No need to build anything here. $result = 0; } else { diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/worker_unittest.py b/Tools/Scripts/lint-webkitpy index ebb4e7a5b..0fbbd73ca 100644..100755 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/worker_unittest.py +++ b/Tools/Scripts/lint-webkitpy @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Copyright (c) 2012 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,29 +27,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import unittest +import os +import sys -from webkitpy.layout_tests.controllers.worker import Worker, WorkerArguments -from webkitpy.tool.mocktool import MockOptions +from webkitpy.thirdparty.autoinstalled.pylint import lint +script_dir = os.path.abspath(os.path.dirname(__file__)) +if not script_dir in sys.path: + sys.path.append(script_dir) -class FakeConnection(object): - def run_message_loop(self): - pass - - def post_message(self, message_name, *message_args): - pass - - -class WorkerTest(unittest.TestCase): - def test_default_platform_in_worker(self): - # This checks that we got a port and didn't raise an exception - # if we didn't specify a port with the --platform flag. - worker_connection = FakeConnection() - worker = Worker(worker_connection, WorkerArguments(1, '/tmp', MockOptions(platform=None, print_options=None, verbose=False, batch_size=0))) - worker._set_up_host_and_port() - self.assertNotEquals(worker._port, None) - - -if __name__ == '__main__': - unittest.main() +pylintrc = os.path.join(script_dir, 'webkitpy', 'pylintrc') +lint.Run(['--rcfile', pylintrc, '-f', 'parseable' ] + sys.argv[1:]) diff --git a/Tools/Scripts/new-run-webkit-tests b/Tools/Scripts/new-run-webkit-tests index 1c3d3b159..c771fd61a 100755 --- a/Tools/Scripts/new-run-webkit-tests +++ b/Tools/Scripts/new-run-webkit-tests @@ -53,7 +53,7 @@ if __name__ == '__main__': # Wrap the NRWT process in the jhbuild environment so DRT or WKTR # doesn't need to do it and their process id as reported by # subprocess.Popen is not jhbuild's. - if '--gtk' in sys.argv[1:]: + if '--gtk' in sys.argv[1:] and os.path.exists(os.path.join(script_dir, '..', '..', 'WebKitBuild', 'Dependencies')): cmd.insert(1, os.path.join(script_dir, '..', 'gtk', 'run-with-jhbuild')) proc = subprocess.Popen(cmd, env=env) diff --git a/Tools/Scripts/prepare-ChangeLog b/Tools/Scripts/prepare-ChangeLog index df687dc2c..cb91c598c 100755 --- a/Tools/Scripts/prepare-ChangeLog +++ b/Tools/Scripts/prepare-ChangeLog @@ -493,18 +493,19 @@ sub generateNewChangeLogs($$$$$$$$$$$) print CHANGE_LOG normalizeLineEndings($description . "\n", $endl) if $description; - $bugDescription = "Need a short description and bug URL (OOPS!)" unless $bugDescription; + $bugDescription = "Need a short description (OOPS!).\n Need the bug URL (OOPS!)." unless $bugDescription; print CHANGE_LOG normalizeLineEndings(" $bugDescription\n", $endl) if $bugDescription; print CHANGE_LOG normalizeLineEndings(" $bugURL\n", $endl) if $bugURL; print CHANGE_LOG normalizeLineEndings("\n", $endl); print CHANGE_LOG normalizeLineEndings(" Reviewed by $reviewer.\n\n", $endl); + print CHANGE_LOG normalizeLineEndings(" Additional information of the change such as approach, rationale. Please add per-function descriptions below (OOPS!).\n\n", $endl); if ($prefix =~ m/WebCore/ || `pwd` =~ m/WebCore/) { if (@$addedRegressionTests) { print CHANGE_LOG normalizeLineEndings(testListForChangeLog(sort @$addedRegressionTests), $endl); } else { - print CHANGE_LOG normalizeLineEndings(" No new tests. (OOPS!)\n\n", $endl); + print CHANGE_LOG normalizeLineEndings(" No new tests (OOPS!).\n\n", $endl); } } diff --git a/Tools/Scripts/run-bindings-tests b/Tools/Scripts/run-bindings-tests index 627f176de..a0785e405 100755 --- a/Tools/Scripts/run-bindings-tests +++ b/Tools/Scripts/run-bindings-tests @@ -51,7 +51,7 @@ def main(argv): from webkitpy.bindings.main import BindingsTests - BindingsTests(reset_results, generators, executive.Executive()).main() + return BindingsTests(reset_results, generators, executive.Executive()).main() if __name__ == '__main__': diff --git a/Tools/Scripts/run-efl-tests b/Tools/Scripts/run-efl-tests new file mode 100755 index 000000000..4a95f50c6 --- /dev/null +++ b/Tools/Scripts/run-efl-tests @@ -0,0 +1,40 @@ +#!/usr/bin/perl -w +# Copyright (C) 2012 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use strict; +use FindBin; +use lib $FindBin::Bin; +use webkitdirs; + +setConfiguration(); + +# Tell CTest to dump gtest output in case of failure. +$ENV{CTEST_OUTPUT_ON_FAILURE} = "1"; + +buildCMakeProjectOrExit(0, "Efl", undef, "test", cmakeBasedPortArguments()); diff --git a/Tools/Scripts/webkitdirs.pm b/Tools/Scripts/webkitdirs.pm index ae634d175..1da09471b 100755 --- a/Tools/Scripts/webkitdirs.pm +++ b/Tools/Scripts/webkitdirs.pm @@ -206,8 +206,12 @@ sub determineBaseProductDir my $buildLocationType = join '', readXcodeUserDefault("CustomBuildLocationType"); # FIXME: Read CustomBuildIntermediatesPath and set OBJROOT accordingly. $baseProductDir = readXcodeUserDefault("CustomBuildProductsPath") if $buildLocationType eq "Absolute"; - $setSharedPrecompsDir = 1; } + + # DeterminedByTargets corresponds to a setting of "Legacy" in Xcode. + # It is the only build location style for which SHARED_PRECOMPS_DIR is not + # overridden when building from within Xcode. + $setSharedPrecompsDir = 1 if $buildLocationStyle ne "DeterminedByTargets"; } if (!defined($baseProductDir)) { @@ -226,9 +230,8 @@ sub determineBaseProductDir } } - if (!defined($baseProductDir)) { # Port-spesific checks failed, use default + if (!defined($baseProductDir)) { # Port-specific checks failed, use default $baseProductDir = "$sourceDir/WebKitBuild"; - undef $setSharedPrecompsDir; } if (isBlackBerry()) { @@ -819,26 +822,39 @@ sub qtFeatureDefaults my $originalCwd = getcwd(); - my $file; + my $file = File::Spec->catfile($qmakepath, "configure.pro"); my @buildArgs; + my $qconfigs; if (@_) { @buildArgs = (@buildArgs, @{$_[0]}); + $qconfigs = $_[1]; my $dir = File::Spec->catfile(productDir(), "Tools", "qmake"); File::Path::mkpath($dir); chdir $dir or die "Failed to cd into " . $dir . "\n"; - $file = File::Spec->catfile($qmakepath, "configure.pro"); } else { # Do a quick check of the features without running the config tests - $file = File::Spec->catfile($qmakepath, "mkspecs", "features", "features.prf"); - push @buildArgs, "CONFIG+=compute_defaults"; + push @buildArgs, "CONFIG+=quick_check"; } - my $defaults = `$qmakecommand @buildArgs $file 2>&1`; + my @defaults = `$qmakecommand @buildArgs -nocache $file 2>&1`; my %qtFeatureDefaults; - while ($defaults =~ m/(\S+?)=(\S+?)/gi) { - $qtFeatureDefaults{$1}=$2; + for (@defaults) { + if (/ DEFINES: /) { + while (/(\S+?)=(\S+?)/gi) { + $qtFeatureDefaults{$1}=$2; + } + } elsif (/ CONFIG:(.*)$/) { + if (@_) { + $$qconfigs = $1; + } + } elsif (/Done computing defaults/) { + print "\n"; + last; + } elsif (@_) { + print $_; + } } chdir $originalCwd; @@ -976,7 +992,7 @@ sub blackberryCMakeArguments() } push @cmakeExtraOptions, "-DCMAKE_SKIP_RPATH='ON'" if isDarwin(); - push @cmakeExtraOptions, "-DENABLE_DRT=1" if $ENV{"ENABLE_DRT"}; + push @cmakeExtraOptions, "-DPUBLIC_BUILD=1" if $ENV{"PUBLIC_BUILD"}; push @cmakeExtraOptions, "-DENABLE_GLES2=1" unless $ENV{"DISABLE_GLES2"}; my @includeSystemDirectories; @@ -2050,6 +2066,9 @@ sub buildAutotoolsProject($@) push @buildArgs, "--disable-debug"; } + # Enable unstable features when building through build-webkit. + push @buildArgs, "--enable-unstable-features"; + # We might need to update jhbuild dependencies. my $needUpdate = 0; if (jhbuildConfigurationChanged()) { @@ -2227,6 +2246,7 @@ sub buildQMakeProjects my ($projects, $clean, @buildParams) = @_; my @buildArgs = (); + my $qconfigs = ""; my $make = qtMakeCommand($qmakebin); my $makeargs = ""; @@ -2273,6 +2293,8 @@ sub buildQMakeProjects } elsif ($passedConfig =~ m/release/i) { push @buildArgs, "CONFIG+=release"; push @buildArgs, "CONFIG-=debug"; + } elsif ($passedConfig) { + die "Build type $passedConfig is not supported with --qt.\n"; } push @buildArgs, "CONFIG-=debug_and_release" if ($passedConfig && isDarwin()); @@ -2281,7 +2303,7 @@ sub buildQMakeProjects File::Path::mkpath($dir); chdir $dir or die "Failed to cd into " . $dir . "\n"; - my %defines = qtFeatureDefaults(\@buildArgs); + my %defines = qtFeatureDefaults(\@buildArgs, \$qconfigs); my $svnRevision = currentSVNRevision(); @@ -2290,6 +2312,8 @@ sub buildQMakeProjects my $pathToDefinesCache = File::Spec->catfile($dir, ".webkit.config"); my $pathToOldDefinesFile = File::Spec->catfile($dir, "defaults.txt"); + # FIXME: Get rid of .webkit.config and defaults.txt and move all the logic to .qmake.cache + # Ease transition to new build layout if (-e $pathToOldDefinesFile) { print "Old build layout detected"; @@ -2349,16 +2373,23 @@ sub buildQMakeProjects File::Path::rmtree($dir); File::Path::mkpath($dir); chdir $dir or die "Failed to cd into " . $dir . "\n"; - - # After removing WebKitBuild directory, we have to call qtFeatureDefaults() - # to run config tests and generate the removed Tools/qmake/.qmake.cache again. - qtFeatureDefaults(\@buildArgs); #} # Still trigger an incremental build $buildHint = "incremental"; } + if ($buildHint eq "incremental") { + my $qmakeDefines = "DEFINES +="; + foreach my $key (sort keys %defines) { + $qmakeDefines .= " \\\n $key=$defines{$key}"; + } + open(QMAKE_CACHE, ">.qmake.cache") or die "Cannot create .qmake.cache!\n"; + print QMAKE_CACHE "CONFIG += webkit_configured $qconfigs\n"; + print QMAKE_CACHE $qmakeDefines."\n"; + close(QMAKE_CACHE); + } + # Save config up-front so we can detect changes to the build config even # when the user re-configures after aborting the build. open(DEFAULTS, ">$pathToDefinesCache"); @@ -2496,6 +2527,7 @@ sub buildChromiumVisualStudioProject($$) } else { $vsInstallDir = "$programFilesPath/Microsoft Visual Studio 8"; } + $vsInstallDir =~ s,\\,/,g; $vsInstallDir = `cygpath "$vsInstallDir"` if isCygwin(); chomp $vsInstallDir; $vcBuildPath = "$vsInstallDir/Common7/IDE/devenv.com"; diff --git a/Tools/Scripts/webkitperl/FeatureList.pm b/Tools/Scripts/webkitperl/FeatureList.pm index 0f033ac55..ca3b9e3d6 100644 --- a/Tools/Scripts/webkitperl/FeatureList.pm +++ b/Tools/Scripts/webkitperl/FeatureList.pm @@ -62,11 +62,11 @@ my ( $datalistSupport, $detailsSupport, $deviceOrientationSupport, + $dialogElementSupport, $directoryUploadSupport, $downloadAttributeSupport, $fileSystemSupport, $filtersSupport, - $fontBoostingSupport, $ftpDirSupport, $fullscreenAPISupport, $gamepadSupport, @@ -116,6 +116,7 @@ my ( $svgFontsSupport, $svgSupport, $systemMallocSupport, + $textAutosizingSupport, $tiledBackingStoreSupport, $touchEventsSupport, $touchIconLoadingSupport, @@ -175,13 +176,13 @@ my @features = ( define => "ENABLE_CSS_SHADERS", default => isAppleMacWebKit(), value => \$cssShadersSupport }, { option => "css-variables", desc => "Toggle CSS Variable support", - define => "ENABLE_CSS_VARIABLES", default => 0, value => \$cssVariablesSupport }, + define => "ENABLE_CSS_VARIABLES", default => isEfl(), value => \$cssVariablesSupport }, { option => "custom-scheme-handler", desc => "Toggle Custom Scheme Handler support", - define => "ENABLE_CUSTOM_SCHEME_HANDLER", default => 0, value => \$customSchemeHandlerSupport }, + define => "ENABLE_CUSTOM_SCHEME_HANDLER", default => (isBlackBerry() || isEfl()), value => \$customSchemeHandlerSupport }, { option => "datalist", desc => "Toggle Datalist support", - define => "ENABLE_DATALIST", default => 0, value => \$datalistSupport }, + define => "ENABLE_DATALIST", default => isEfl(), value => \$datalistSupport }, { option => "data-transfer-items", desc => "Toggle Data Transfer Items support", define => "ENABLE_DATA_TRANSFER_ITEMS", default => 0, value => \$dataTransferItemsSupport }, @@ -192,7 +193,10 @@ my @features = ( { option => "device-orientation", desc => "Toggle Device Orientation support", define => "ENABLE_DEVICE_ORIENTATION", default => isBlackBerry(), value => \$deviceOrientationSupport }, - { option => "directory-upload", desc => "Toogle Directory Upload support", + { option => "dialog", desc => "Toggle Dialog Element support", + define => "ENABLE_DIALOG_ELEMENT", default => 0, value => \$dialogElementSupport }, + + { option => "directory-upload", desc => "Toggle Directory Upload support", define => "ENABLE_DIRECTORY_UPLOAD", default => 0, value => \$directoryUploadSupport }, { option => "download-attribute", desc => "Toggle Download Attribute support", @@ -204,9 +208,6 @@ my @features = ( { option => "filters", desc => "Toggle Filters support", define => "ENABLE_FILTERS", default => (isAppleWebKit() || isGtk() || isQt() || isEfl() || isBlackBerry()), value => \$filtersSupport }, - { option => "font-boosting", desc => "Toggle Font Boosting support", - define => "ENABLE_FONT_BOOSTING", default => 0, value => \$fontBoostingSupport }, - { option => "ftpdir", desc => "Toggle FTP Directory support", define => "ENABLE_FTPDIR", default => !isWinCE(), value => \$ftpDirSupport }, @@ -214,7 +215,7 @@ my @features = ( define => "ENABLE_FULLSCREEN_API", default => (isAppleMacWebKit() || isEfl() || isGtk() || isBlackBerry() || isQt()), value => \$fullscreenAPISupport }, { option => "gamepad", desc => "Toggle Gamepad support", - define => "ENABLE_GAMEPAD", default => 0, value => \$gamepadSupport }, + define => "ENABLE_GAMEPAD", default => (isEfl() || isGtk()), value => \$gamepadSupport }, { option => "geolocation", desc => "Toggle Geolocation support", define => "ENABLE_GEOLOCATION", default => (isAppleWebKit() || isGtk() || isBlackBerry()), value => \$geolocationSupport }, @@ -292,7 +293,7 @@ my @features = ( define => "ENABLE_MHTML", default => 0, value => \$mhtmlSupport }, { option => "microdata", desc => "Toggle Microdata support", - define => "ENABLE_MICRODATA", default => 0, value => \$microdataSupport }, + define => "ENABLE_MICRODATA", default => (isEfl() || isBlackBerry()), value => \$microdataSupport }, { option => "mutation-observers", desc => "Toggle Mutation Observers support", define => "ENABLE_MUTATION_OBSERVERS", default => 1, value => \$mutationObserversSupport }, @@ -319,7 +320,7 @@ my @features = ( define => "ENABLE_QUOTA", default => 0, value => \$quotaSupport }, { option => "register-protocol-handler", desc => "Toggle Register Protocol Handler support", - define => "ENABLE_REGISTER_PROTOCOL_HANDLER", default => isEfl(), value => \$registerProtocolHandlerSupport }, + define => "ENABLE_REGISTER_PROTOCOL_HANDLER", default => (isBlackBerry() || isEfl()), value => \$registerProtocolHandlerSupport }, { option => "request-animation-frame", desc => "Toggle Request Animation Frame support", define => "ENABLE_REQUEST_ANIMATION_FRAME", default => (isAppleMacWebKit() || isGtk() || isEfl() || isBlackBerry()), value => \$requestAnimationFrameSupport }, @@ -337,7 +338,7 @@ my @features = ( define => "ENABLE_SQL_DATABASE", default => 1, value => \$sqlDatabaseSupport }, { option => "style-scoped", desc => "Toggle Style Scoped support", - define => "ENABLE_STYLE_SCOPED", default => 0, value => \$styleScopedSupport }, + define => "ENABLE_STYLE_SCOPED", default => isBlackBerry(), value => \$styleScopedSupport }, { option => "svg", desc => "Toggle SVG support", define => "ENABLE_SVG", default => 1, value => \$svgSupport }, @@ -351,6 +352,9 @@ my @features = ( { option => "system-malloc", desc => "Toggle system allocator instead of TCmalloc", define => "USE_SYSTEM_MALLOC", default => isWinCE(), value => \$systemMallocSupport }, + { option => "text-autosizing", desc => "Toggle Text Autosizing support", + define => "ENABLE_TEXT_AUTOSIZING", default => 0, value => \$textAutosizingSupport }, + { option => "tiled-backing-store", desc => "Toggle Tiled Backing Store support", define => "WTF_USE_TILED_BACKING_STORE", default => isQt(), value => \$tiledBackingStoreSupport }, diff --git a/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer.py b/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer.py index 8ae0bb9b9..d244045ac 100644 --- a/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer.py +++ b/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer.py @@ -52,6 +52,7 @@ def _baseline_search_hypergraph(host): _VIRTUAL_PORTS = { 'mac-future': ['LayoutTests/platform/mac-future', 'LayoutTests/platform/mac', 'LayoutTests'], + 'win-future': ['LayoutTests/platform/win-future', 'LayoutTests/platform/win', 'LayoutTests'], 'qt-unknown': ['LayoutTests/platform/qt-unknown', 'LayoutTests/platform/qt', 'LayoutTests'], } diff --git a/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer_unittest.py b/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer_unittest.py index 414cc06df..9ba6ff1f2 100644 --- a/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer_unittest.py +++ b/Tools/Scripts/webkitpy/common/checkout/baselineoptimizer_unittest.py @@ -119,6 +119,21 @@ class BaselineOptimizerTest(unittest.TestCase): 'LayoutTests/platform/qt': '462d03b9c025db1b0392d7453310dbee5f9a9e74', }) + def test_win_does_not_drop_to_win_7sp0(self): + self._assertOptimization({ + 'LayoutTests/platform/win': '1', + 'LayoutTests/platform/mac': '2', + 'LayoutTests/platform/gtk': '3', + 'LayoutTests/platform/qt': '4', + 'LayoutTests/platform/chromium': '5', + }, { + 'LayoutTests/platform/win': '1', + 'LayoutTests/platform/mac': '2', + 'LayoutTests/platform/gtk': '3', + 'LayoutTests/platform/qt': '4', + 'LayoutTests/platform/chromium': '5', + }) + def test_common_directory_includes_root(self): # Note: The resulting directories are "wrong" in the sense that # enacting this plan would change semantics. However, this test case @@ -138,7 +153,7 @@ class BaselineOptimizerTest(unittest.TestCase): 'LayoutTests/platform/chromium-win': '23a30302a6910f8a48b1007fa36f3e3158341834', 'LayoutTests': '9c876f8c3e4cc2aef9519a6c1174eb3432591127', 'LayoutTests/platform/chromium-mac': '23a30302a6910f8a48b1007fa36f3e3158341834', - 'LayoutTests/platform/chromium-mac': '23a30302a6910f8a48b1007fa36f3e3158341834', + 'LayoutTests/platform/chromium': '1', }, { 'LayoutTests/platform/chromium': '23a30302a6910f8a48b1007fa36f3e3158341834', 'LayoutTests': '9c876f8c3e4cc2aef9519a6c1174eb3432591127', @@ -153,7 +168,6 @@ class BaselineOptimizerTest(unittest.TestCase): 'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74', 'LayoutTests/platform/mac': '5daa78e55f05d9f0d1bb1f32b0cd1bc3a01e9364', 'LayoutTests/platform/chromium-win-xp': '462d03b9c025db1b0392d7453310dbee5f9a9e74', - 'LayoutTests/platform/chromium-mac-leopard': '65e7d42f8b4882b29d46dc77bb879dd41bc074dc', 'LayoutTests/platform/mac-lion': '7ad045ece7c030e2283c5d21d9587be22bcba56e', 'LayoutTests/platform/chromium-win': 'f83af9732ce74f702b8c9c4a3d9a4c6636b8d3bd', 'LayoutTests/platform/win-xp': '5b1253ef4d5094530d5f1bc6cdb95c90b446bec7', @@ -162,7 +176,6 @@ class BaselineOptimizerTest(unittest.TestCase): 'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74', 'LayoutTests/platform/mac': '5daa78e55f05d9f0d1bb1f32b0cd1bc3a01e9364', 'LayoutTests/platform/chromium-win-xp': '462d03b9c025db1b0392d7453310dbee5f9a9e74', - 'LayoutTests/platform/chromium-mac-leopard': '65e7d42f8b4882b29d46dc77bb879dd41bc074dc', 'LayoutTests/platform/mac-lion': '7ad045ece7c030e2283c5d21d9587be22bcba56e', 'LayoutTests/platform/chromium-win': 'f83af9732ce74f702b8c9c4a3d9a4c6636b8d3bd', 'LayoutTests/platform/win-xp': '5b1253ef4d5094530d5f1bc6cdb95c90b446bec7', diff --git a/Tools/Scripts/webkitpy/common/checkout/changelog.py b/Tools/Scripts/webkitpy/common/checkout/changelog.py index f30fd2c2d..ae7b71fc0 100644 --- a/Tools/Scripts/webkitpy/common/checkout/changelog.py +++ b/Tools/Scripts/webkitpy/common/checkout/changelog.py @@ -327,7 +327,7 @@ class ChangeLog(object): def update_with_unreviewed_message(self, message): first_boilerplate_line_regexp = re.compile( - "%sNeed a short description and bug URL \(OOPS!\)" % self._changelog_indent) + "%sNeed a short description \(OOPS!\)\." % self._changelog_indent) removing_boilerplate = False # inplace=1 creates a backup file and re-directs stdout to the file for line in fileinput.FileInput(self.path, inplace=1): @@ -345,12 +345,33 @@ class ChangeLog(object): print line, def set_reviewer(self, reviewer): - # inplace=1 creates a backup file and re-directs stdout to the file - for line in fileinput.FileInput(self.path, inplace=1): - # Trailing comma suppresses printing newline - print line.replace("NOBODY (OOPS!)", reviewer.encode("utf-8")), + latest_entry = self.latest_entry() + latest_entry_contents = latest_entry.contents() + reviewer_text = latest_entry.reviewer() + found_nobody = re.search("NOBODY\s*\(OOPS!\)", latest_entry_contents, re.MULTILINE) + + if not found_nobody and not reviewer_text: + bug_url_number_of_items = len(re.findall(config_urls.bug_url_long, latest_entry_contents, re.MULTILINE)) + bug_url_number_of_items += len(re.findall(config_urls.bug_url_short, latest_entry_contents, re.MULTILINE)) + for line in fileinput.FileInput(self.path, inplace=1): + found_bug_url = re.search(config_urls.bug_url_long, line) + if not found_bug_url: + found_bug_url = re.search(config_urls.bug_url_short, line) + print line, + if found_bug_url: + if bug_url_number_of_items == 1: + print "\n Reviewed by %s." % (reviewer.encode("utf-8")) + bug_url_number_of_items -= 1 + else: + # inplace=1 creates a backup file and re-directs stdout to the file + for line in fileinput.FileInput(self.path, inplace=1): + # Trailing comma suppresses printing newline + print line.replace("NOBODY (OOPS!)", reviewer.encode("utf-8")), def set_short_description_and_bug_url(self, short_description, bug_url): - message = "%s\n %s" % (short_description, bug_url) + message = "%s\n%s%s" % (short_description, self._changelog_indent, bug_url) + bug_boilerplate = "%sNeed the bug URL (OOPS!).\n" % self._changelog_indent for line in fileinput.FileInput(self.path, inplace=1): - print line.replace("Need a short description and bug URL (OOPS!)", message.encode("utf-8")), + line = line.replace("Need a short description (OOPS!).", message.encode("utf-8")) + if line != bug_boilerplate: + print line, diff --git a/Tools/Scripts/webkitpy/common/checkout/changelog_unittest.py b/Tools/Scripts/webkitpy/common/checkout/changelog_unittest.py index d2040bf2e..9591744d1 100644 --- a/Tools/Scripts/webkitpy/common/checkout/changelog_unittest.py +++ b/Tools/Scripts/webkitpy/common/checkout/changelog_unittest.py @@ -489,15 +489,54 @@ class ChangeLogTest(unittest.TestCase): # FIXME: We really should be getting this from prepare-ChangeLog itself. _new_entry_boilerplate = '''2009-08-19 Eric Seidel <eric@webkit.org> - Need a short description and bug URL (OOPS!) + Need a short description (OOPS!). + Need the bug URL (OOPS!). Reviewed by NOBODY (OOPS!). * Scripts/bugzilla-tool: ''' + _new_entry_boilerplate_with_bugurl = '''2009-08-19 Eric Seidel <eric@webkit.org> + + Need a short description (OOPS!). + https://bugs.webkit.org/show_bug.cgi?id=12345 + + Reviewed by NOBODY (OOPS!). + + * Scripts/bugzilla-tool: +''' + + _new_entry_boilerplate_with_multiple_bugurl = '''2009-08-19 Eric Seidel <eric@webkit.org> + + Need a short description (OOPS!). + https://bugs.webkit.org/show_bug.cgi?id=12345 + http://webkit.org/b/12345 + + Reviewed by NOBODY (OOPS!). + + * Scripts/bugzilla-tool: +''' + + _new_entry_boilerplate_without_reviewer_line = '''2009-08-19 Eric Seidel <eric@webkit.org> + + Need a short description (OOPS!). + https://bugs.webkit.org/show_bug.cgi?id=12345 + + * Scripts/bugzilla-tool: +''' + + _new_entry_boilerplate_without_reviewer_multiple_bugurl = '''2009-08-19 Eric Seidel <eric@webkit.org> + + Need a short description (OOPS!). + https://bugs.webkit.org/show_bug.cgi?id=12345 + http://webkit.org/b/12345 + + * Scripts/bugzilla-tool: +''' + def test_set_reviewer(self): - changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate_with_bugurl, self._example_changelog) changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) reviewer_name = 'Test Reviewer' ChangeLog(changelog_path).set_reviewer(reviewer_name) @@ -506,14 +545,41 @@ class ChangeLogTest(unittest.TestCase): os.remove(changelog_path) self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) + changelog_contents_without_reviewer_line = u"%s\n%s" % (self._new_entry_boilerplate_without_reviewer_line, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents_without_reviewer_line.encode("utf-8")) + ChangeLog(changelog_path).set_reviewer(reviewer_name) + actual_contents = self._read_file_contents(changelog_path, "utf-8") + os.remove(changelog_path) + self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) + + changelog_contents_without_reviewer_line = u"%s\n%s" % (self._new_entry_boilerplate_without_reviewer_multiple_bugurl, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents_without_reviewer_line.encode("utf-8")) + ChangeLog(changelog_path).set_reviewer(reviewer_name) + actual_contents = self._read_file_contents(changelog_path, "utf-8") + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate_with_multiple_bugurl, self._example_changelog) + expected_contents = changelog_contents.replace('NOBODY (OOPS!)', reviewer_name) + os.remove(changelog_path) + self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) + def test_set_short_description_and_bug_url(self): - changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate_with_bugurl, self._example_changelog) changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) short_description = "A short description" bug_url = "http://example.com/b/2344" ChangeLog(changelog_path).set_short_description_and_bug_url(short_description, bug_url) actual_contents = self._read_file_contents(changelog_path, "utf-8") expected_message = "%s\n %s" % (short_description, bug_url) - expected_contents = changelog_contents.replace("Need a short description and bug URL (OOPS!)", expected_message) + expected_contents = changelog_contents.replace("Need a short description (OOPS!).", expected_message) + os.remove(changelog_path) + self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) + + changelog_contents = u"%s\n%s" % (self._new_entry_boilerplate, self._example_changelog) + changelog_path = self._write_tmp_file_with_contents(changelog_contents.encode("utf-8")) + short_description = "A short description 2" + bug_url = "http://example.com/b/2345" + ChangeLog(changelog_path).set_short_description_and_bug_url(short_description, bug_url) + actual_contents = self._read_file_contents(changelog_path, "utf-8") + expected_message = "%s\n %s" % (short_description, bug_url) + expected_contents = changelog_contents.replace("Need a short description (OOPS!).\n Need the bug URL (OOPS!).", expected_message) os.remove(changelog_path) self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) diff --git a/Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py b/Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py index 78d25cb31..e9c2cddda 100644 --- a/Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py +++ b/Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py @@ -259,5 +259,5 @@ class CheckoutTest(unittest.TestCase): mock_patch = Mock() mock_patch.contents = lambda: "foo" mock_patch.reviewer = lambda: None - expected_stderr = "MOCK run_command: ['svn-apply', '--force'], cwd=/mock-checkout\n" + expected_stderr = "MOCK run_command: ['svn-apply', '--force'], cwd=/mock-checkout, input=foo\n" OutputCapture().assert_outputs(self, checkout.apply_patch, [mock_patch], expected_stderr=expected_stderr) diff --git a/Tools/Scripts/webkitpy/common/checksvnconfigfile.py b/Tools/Scripts/webkitpy/common/checksvnconfigfile.py new file mode 100644 index 000000000..e6165f64b --- /dev/null +++ b/Tools/Scripts/webkitpy/common/checksvnconfigfile.py @@ -0,0 +1,65 @@ +# Copyright (C) 2012 Balazs Ankes (bank@inf.u-szeged.hu) University of Szeged +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is used by: +# webkitpy/tool/steps/addsvnmimetypeforpng.py +# webkitpy/style/checkers/png.py + +import os +import re + + +def check(host, fs): + """ + check the svn config file + return with three logical value: + is svn config file missing, is auto-props missing, is the svn:mime-type for png missing + """ + + cfg_file_path = config_file_path(host, fs) + + try: + config_file = fs.read_text_file(cfg_file_path) + except IOError: + return (True, True, True) + + errorcode_autoprop = not re.search("^\s*enable-auto-props\s*=\s*yes", config_file, re.MULTILINE) + errorcode_png = not re.search("^\s*\*\.png\s*=\s*svn:mime-type=image/png", config_file, re.MULTILINE) + + return (False, errorcode_autoprop, errorcode_png) + + +def config_file_path(host, fs): + if host.platform.is_win(): + config_file_path = fs.join(os.environ['APPDATA'], "Subversion", "config") + else: + config_file_path = fs.join(fs.expanduser("~"), ".subversion", "config") + return config_file_path + + +def errorstr_autoprop(config_file_path): + return 'Have to enable auto props in the subversion config file (%s "enable-auto-props = yes"). ' % config_file_path + + +def errorstr_png(config_file_path): + return 'Have to set the svn:mime-type in the subversion config file (%s "*.png = svn:mime-type=image/png").' % config_file_path diff --git a/Tools/Scripts/webkitpy/common/config/committers.py b/Tools/Scripts/webkitpy/common/config/committers.py index cdbd3e15b..f31510022 100644 --- a/Tools/Scripts/webkitpy/common/config/committers.py +++ b/Tools/Scripts/webkitpy/common/config/committers.py @@ -110,10 +110,8 @@ contributors_who_are_not_committers = [ Contributor("Adam Kallai", "kadam@inf.u-szeged.hu", 'kadam'), Contributor("Aharon Lanin", "aharon@google.com"), Contributor("Alan Stearns", "stearns@adobe.com", 'astearns'), - Contributor("Alec Flett", ["alecflett@chromium.org", "alecflett@google.com"], "alecf"), Contributor("Alexandre Elias", "aelias@chromium.org"), Contributor("Alexey Marinichev", ["amarinichev@chromium.org", "amarinichev@google.com"], "amarinichev"), - Contributor("Allan Sandfeld Jensen", ["allan.jensen@nokia.com", "kde@carewolf.com", "sandfeld@kde.org"], "carewolf"), Contributor("Andras Piroska", "pandras@inf.u-szeged.hu", "andris88"), Contributor("Anne van Kesteren", "annevankesteren+webkit@gmail.com", "annevk"), Contributor("Annie Sullivan", "sullivan@chromium.org", "annie"), @@ -122,7 +120,6 @@ contributors_who_are_not_committers = [ Contributor("Brian Salomon", "bsalomon@google.com"), Contributor("Commit Queue", "commit-queue@webkit.org"), Contributor("Daniel Sievers", "sievers@chromium.org"), - Contributor("Dave Tharp", "dtharp@codeaurora.org", "dtharp"), Contributor("David Barr", "davidbarr@chromium.org", "barrbrain"), Contributor("David Dorwin", "ddorwin@chromium.org", "ddorwin"), Contributor("David Reveman", "reveman@chromium.org", "reveman"), @@ -136,7 +133,6 @@ contributors_who_are_not_committers = [ Contributor("Gregg Tavares", ["gman@google.com", "gman@chromium.org"], "gman"), Contributor("Hao Zheng", "zhenghao@chromium.org"), Contributor("Ian Hickson", "ian@hixie.ch", "hixie"), - Contributor("Ian Vollick", "vollick@chromium.org", "vollick"), Contributor("Janos Badics", "jbadics@inf.u-szeged.hu", 'dicska'), Contributor("Jing Zhao", "jingzhao@chromium.org"), Contributor("John Bates", ["jbates@google.com", "jbates@chromium.org"], "jbates"), @@ -179,11 +175,13 @@ committers_unable_to_review = [ Committer("Adam Langley", "agl@chromium.org", "agl"), Committer("Ademar de Souza Reis Jr", ["ademar.reis@gmail.com", "ademar@webkit.org"], "ademar"), Committer("Albert J. Wong", "ajwong@chromium.org"), + Committer("Alec Flett", ["alecflett@chromium.org", "alecflett@google.com"], "alecf"), Committer(u"Alexander F\u00e6r\u00f8y", ["ahf@0x90.dk", "alexander.faeroy@nokia.com"], "ahf"), Committer("Alexander Kellett", ["lypanov@mac.com", "a-lists001@lypanov.net", "lypanov@kde.org"], "lypanov"), Committer("Alexander Pavlov", "apavlov@chromium.org", "apavlov"), Committer("Alexandru Chiculita", "achicu@adobe.com", "achicu"), Committer("Alice Boxhall", "aboxhall@chromium.org", "aboxhall"), + Committer("Allan Sandfeld Jensen", ["allan.jensen@nokia.com", "kde@carewolf.com", "sandfeld@kde.org"], "carewolf"), Committer("Alok Priyadarshi", "alokp@chromium.org", "alokp"), Committer("Ami Fischman", ["fischman@chromium.org", "fischman@google.com"], "fischman"), Committer("Amruth Raj", "amruthraj@motorola.com", "amruthraj"), @@ -224,6 +222,7 @@ committers_unable_to_review = [ Committer("Dana Jansens", "danakj@chromium.org", "danakj"), Committer("Daniel Cheng", "dcheng@chromium.org", "dcheng"), Committer("Dave Barton", "dbarton@mathscribe.com", "dbarton"), + Committer("Dave Tharp", "dtharp@codeaurora.org", "dtharp"), Committer("David Grogan", ["dgrogan@chromium.org", "dgrogan@google.com"], "dgrogan"), Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"], "catfishman"), Committer("Diego Gonzalez", ["diegohcg@webkit.org", "diego.gonzalez@openbossa.org"], "diegohcg"), @@ -256,6 +255,7 @@ committers_unable_to_review = [ Committer("Hironori Bono", "hbono@chromium.org", "hbono"), Committer("Helder Correia", "helder.correia@nokia.com", "helder"), Committer("Hin-Chung Lam", ["hclam@google.com", "hclam@chromium.org"]), + Committer("Ian Vollick", "vollick@chromium.org", "vollick"), Committer("Igor Trindade Oliveira", ["igor.oliveira@webkit.org", "igor.o@sisa.samsung.com"], "igoroliveira"), Committer("Ilya Sherman", "isherman@chromium.org", "isherman"), Committer("Ilya Tikhonovsky", "loislo@chromium.org", "loislo"), @@ -267,6 +267,7 @@ committers_unable_to_review = [ Committer("James Kozianski", ["koz@chromium.org", "koz@google.com"], "koz"), Committer("James Simonsen", "simonjam@chromium.org", "simonjam"), Committer("Jarred Nicholls", ["jarred@webkit.org", "jarred@sencha.com"], "jarrednicholls"), + Committer("Jason Liu", ["jason.liu@torchmobile.com.cn", "jasonliuwebkit@gmail.com"], "jasonliu"), Committer("Jay Civelli", "jcivelli@chromium.org", "jcivelli"), Committer("Jeff Miller", "jeffm@apple.com", "jeffm7"), Committer("Jeffrey Pfau", ["jeffrey@endrift.com", "jpfau@apple.com"], "jpfau"), @@ -277,6 +278,7 @@ committers_unable_to_review = [ Committer("Jesus Sanchez-Palencia", ["jesus@webkit.org", "jesus.palencia@openbossa.org"], "jeez_"), Committer("Jia Pu", "jpu@apple.com"), Committer("Jochen Eisinger", "jochen@chromium.org", "jochen__"), + Committer("Joe Thomas", "joethomas@motorola.com", "joethomas"), Committer("John Abd-El-Malek", "jam@chromium.org", "jam"), Committer("John Gregg", ["johnnyg@google.com", "johnnyg@chromium.org"], "johnnyg"), Committer("John Knottenbelt", "jknotten@chromium.org", "jknotten"), @@ -312,7 +314,6 @@ committers_unable_to_review = [ Committer("Mahesh Kulkarni", ["mahesh.kulkarni@nokia.com", "maheshk@webkit.org"], "maheshk"), Committer("Marcus Voltis Bulach", "bulach@chromium.org"), Committer("Mario Sanchez Prada", ["msanchez@igalia.com", "mario@webkit.org"], "msanchez"), - Committer("Mark Hahnenberg", "mhahnenberg@apple.com"), Committer("Mary Wu", ["mary.wu@torchmobile.com.cn", "wwendy2007@gmail.com"], "marywu"), Committer("Matt Delaney", "mdelaney@apple.com"), Committer("Matt Lilek", ["mlilek@apple.com", "webkit@mattlilek.com", "pewtermoose@webkit.org"], "pewtermoose"), @@ -365,7 +366,6 @@ committers_unable_to_review = [ Committer("Steve Lacey", "sjl@chromium.org", "stevela"), Committer("Takashi Toyoshima", "toyoshim@chromium.org", "toyoshim"), Committer("Thomas Sepez", "tsepez@chromium.org", "tsepez"), - Committer("Tim Horton", "timothy_horton@apple.com", "thorton"), Committer("Tom Zakrajsek", "tomz@codeaurora.org", "tomz"), Committer("Tommy Widenflycht", "tommyw@google.com", "tommyw"), Committer("Trey Matteson", "trey@usa.net", "trey"), @@ -483,6 +483,7 @@ reviewers_list = [ Reviewer("Levi Weintraub", ["leviw@chromium.org", "leviw@google.com", "lweintraub@apple.com"], "leviw"), Reviewer("Luiz Agostini", ["luiz@webkit.org", "luiz.agostini@openbossa.org"], "lca"), Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), + Reviewer("Mark Hahnenberg", "mhahnenberg@apple.com", "mhahnenberg"), Reviewer("Mark Rowe", "mrowe@apple.com", "bdash"), Reviewer("Martin Robinson", ["mrobinson@webkit.org", "mrobinson@igalia.com", "martin.james.robinson@gmail.com"], "mrobinson"), Reviewer("Michael Saboff", "msaboff@apple.com", "msaboff"), @@ -507,6 +508,7 @@ reviewers_list = [ Reviewer("Steve Falkenburg", "sfalken@apple.com", "sfalken"), Reviewer("Tim Omernick", "timo@apple.com"), Reviewer("Timothy Hatcher", ["timothy@apple.com", "timothy@hatcher.name"], "xenon"), + Reviewer("Tim Horton", "timothy_horton@apple.com", "thorton"), Reviewer("Tony Chang", "tony@chromium.org", "tony^work"), Reviewer("Tony Gentilcore", "tonyg@chromium.org", "tonyg-cr"), Reviewer(u"Tor Arne Vestb\u00f8", ["vestbo@webkit.org", "tor.arne.vestbo@nokia.com"], "torarne"), diff --git a/Tools/Scripts/webkitpy/common/config/watchlist b/Tools/Scripts/webkitpy/common/config/watchlist index 5d5f59061..c11656322 100755 --- a/Tools/Scripts/webkitpy/common/config/watchlist +++ b/Tools/Scripts/webkitpy/common/config/watchlist @@ -157,13 +157,35 @@ }, "MathML": { "filename": r"(Source|LayoutTests|Websites)/.*mathml", - } + }, + "Editing": { + "filename": r"Source/WebCore/editing/", + }, + "BlackBerry": { + "filename": r"Source/WebKit/blackberry/" + r"|Source/WebCore/page/blackberry" + r"|Source/WebCore/history/blackberry" + r"|Source/WebCore/plugins/blackberry" + r"|Source/WebCore/editing/blackberry" + r"|Source/WebCore/Resources/blackberry" + r"|Source/WebCore/platform/image-decoders/blackberry" + r"|Source/WebCore/platform/blackberry" + r"|Source/WebCore/platform/text/blackberry" + r"|Source/WebCore/platform/network/blackberry" + r"|Source/WebCore/platform/graphics/blackberry" + r"|Source/WTF/wtf/blackberry" + r"|ManualTests/blackberry" + r"|Tools/DumpRenderTree/blackberry" + r"|LayoutTests/platform/blackberry", + }, + }, "CC_RULES": { # Note: All email addresses listed must be registered with bugzilla. # Specifically, levin@chromium.org and levin+threading@chromium.org are # two different accounts as far as bugzilla is concerned. "AppleMacPublicApi": [ "timothy@apple.com" ], + "BlackBerry": [ "mifenton@rim.com" ], "CMake": [ "rakuco@webkit.org", ], "CSS": [ "alexis.menard@openbossa.org", "macpherson@chromium.org", "cmarcelo@webkit.org" ], "ChromiumDumpRenderTree": [ "tkent@chromium.org", ], @@ -171,7 +193,8 @@ "ChromiumPublicApi": [ "abarth@webkit.org", "dglazkov@chromium.org", "fishd@chromium.org", "jamesr@chromium.org", "tkent+wkapi@chromium.org" ], "DOMAttributes": [ "cmarcelo@webkit.org", ], "EFL": [ "rakuco@webkit.org", ], - "Forms": [ "tkent@chromium.org", ], + "Editing": [ "mifenton@rim.com" ], + "Forms": [ "tkent@chromium.org", "mifenton@rim.com" ], "FrameLoader": [ "abarth@webkit.org", "japhet@chromium.org", "jochen@chromium.org" ], "GStreamerGraphics": [ "alexis.menard@openbossa.org", "pnormand@igalia.com", "gns@gnome.org", "mrobinson@webkit.org" ], "GtkWebKit2PublicAPI": [ "cgarcia@igalia.com", "gns@gnome.org", "mrobinson@webkit.org" ], diff --git a/Tools/Scripts/webkitpy/common/host.py b/Tools/Scripts/webkitpy/common/host.py index 083120227..53889657b 100644 --- a/Tools/Scripts/webkitpy/common/host.py +++ b/Tools/Scripts/webkitpy/common/host.py @@ -138,6 +138,11 @@ class Host(SystemHost): def checkout(self): return self._checkout + def buildbot_for_builder_name(self, name): + if self.port_factory.get_from_builder_name(name).is_chromium(): + return self.chromium_buildbot() + return self.buildbot + @memoized def chromium_buildbot(self): return ChromiumBuildBot() diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py index 159b0077d..adb5a3d2c 100644 --- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py +++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py @@ -33,6 +33,7 @@ import urllib import urllib2 import webkitpy.common.config.urls as config_urls +from webkitpy.common.memoized import memoized from webkitpy.common.net.failuremap import FailureMap from webkitpy.common.net.layouttestresults import LayoutTestResults from webkitpy.common.net.networktransaction import NetworkTransaction @@ -65,6 +66,31 @@ class Builder(object): def accumulated_results_url(self): return None + def latest_layout_test_results_url(self): + return self.accumulated_results_url() or self.latest_cached_build().results_url(); + + @memoized + def latest_layout_test_results(self): + return self.fetch_layout_test_results(self.latest_layout_test_results_url()) + + def _fetch_file_from_results(self, results_url, file_name): + # It seems this can return None if the url redirects and then returns 404. + result = urllib2.urlopen("%s/%s" % (results_url, file_name)) + if not result: + return None + # urlopen returns a file-like object which sometimes works fine with str() + # but sometimes is a addinfourl object. In either case calling read() is correct. + return result.read() + + def fetch_layout_test_results(self, results_url): + # FIXME: This should cache that the result was a 404 and stop hitting the network. + results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json")) + if not results_file: + results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html")) + + # results_from_string accepts either ORWT html or NRWT json. + return LayoutTestResults.results_from_string(results_file) + def url_encoded_name(self): return urllib.quote(self._name) @@ -120,6 +146,8 @@ class Builder(object): def _revision_and_build_for_filename(self, filename): # Example: "r47483 (1)/" or "r47483 (1).zip" match = self.file_name_regexp.match(filename) + if not match: + return None return (int(match.group("revision")), int(match.group("build_number"))) def _fetch_revision_to_build_map(self): @@ -135,10 +163,18 @@ class Builder(object): except urllib2.HTTPError, error: if error.code != 404: raise + _log.debug("Revision/build list failed to load.") result_files = [] + return dict(self._file_info_list_to_revision_to_build_list(result_files)) + def _file_info_list_to_revision_to_build_list(self, file_info_list): # This assumes there was only one build per revision, which is false but we don't care for now. - return dict([self._revision_and_build_for_filename(file_info["filename"]) for file_info in result_files]) + revisions_and_builds = [] + for file_info in file_info_list: + revision_and_build = self._revision_and_build_for_filename(file_info["filename"]) + if revision_and_build: + revisions_and_builds.append(revision_and_build) + return revisions_and_builds def _revision_to_build_map(self): if not self._revision_to_build_number: @@ -219,7 +255,6 @@ class Build(object): self._number = build_number self._revision = revision self._is_green = is_green - self._layout_test_results = None @staticmethod def build_url(builder, build_number): @@ -235,27 +270,9 @@ class Build(object): def results_zip_url(self): return "%s.zip" % self.results_url() - def _fetch_file_from_results(self, file_name): - # It seems this can return None if the url redirects and then returns 404. - result = urllib2.urlopen("%s/%s" % (self.results_url(), file_name)) - if not result: - return None - # urlopen returns a file-like object which sometimes works fine with str() - # but sometimes is a addinfourl object. In either case calling read() is correct. - return result.read() - + @memoized def layout_test_results(self): - if self._layout_test_results: - return self._layout_test_results - - # FIXME: This should cache that the result was a 404 and stop hitting the network. - results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results("full_results.json")) - if not results_file: - results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results("results.html")) - - # results_from_string accepts either ORWT html or NRWT json. - self._layout_test_results = LayoutTestResults.results_from_string(results_file) - return self._layout_test_results + return self._builder.fetch_layout_test_results(self.results_url()) def builder(self): return self._builder diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_mock.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_mock.py index 966fd5fc6..f8ec49b7b 100644 --- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_mock.py +++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_mock.py @@ -27,6 +27,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +class MockBuild(object): + def __init__(self, build_number, revision, is_green): + self._number = build_number + self._revision = revision + self._is_green = is_green + class MockBuilder(object): def __init__(self, name): self._name = name @@ -34,12 +40,18 @@ class MockBuilder(object): def name(self): return self._name + def build(self, build_number): + return MockBuild(build_number=build_number, revision=1234, is_green=False) + def results_url(self): return "http://example.com/builders/%s/results" % self.name() def accumulated_results_url(self): return "http://example.com/f/builders/%s/results/layout-test-results" % self.name() + def latest_layout_test_results_url(self): + return self.accumulated_results_url() + def force_build(self, username, comments): log("MOCK: force_build: name=%s, username=%s, comments=%s" % ( self._name, username, comments)) diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py index 355786ae0..69f864889 100644 --- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py +++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py @@ -48,7 +48,10 @@ class BuilderTest(unittest.TestCase): is_green=build_number < 4 ) results = [self._mock_test_result(testname) for testname in failure(build_number)] - build._layout_test_results = LayoutTestResults(results) + layout_test_results = LayoutTestResults(results) + def mock_layout_test_results(): + return layout_test_results + build.layout_test_results = mock_layout_test_results return build self.builder._fetch_build = _mock_fetch_build @@ -57,6 +60,11 @@ class BuilderTest(unittest.TestCase): self.builder = Builder(u"Test Builder \u2661", self.buildbot) self._install_fetch_build(lambda build_number: ["test1", "test2"]) + def test_latest_layout_test_results(self): + self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults([self._mock_test_result(testname) for testname in ["test1", "test2"]]) + self.builder.accumulated_results_url = lambda: "http://dummy_url.org" + self.assertTrue(self.builder.latest_layout_test_results()) + def test_find_regression_window(self): regression_window = self.builder.find_regression_window(self.builder.build(10)) self.assertEqual(regression_window.build_before_failure().revision(), 1003) @@ -112,10 +120,20 @@ class BuilderTest(unittest.TestCase): expectations = { "r47483 (1)/" : (47483, 1), "r47483 (1).zip" : (47483, 1), + "random junk": None, } for filename, revision_and_build in expectations.items(): self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) + def test_file_info_list_to_revision_to_build_list(self): + file_info_list = [ + {"filename": "r47483 (1)/"}, + {"filename": "r47483 (1).zip"}, + {"filename": "random junk"}, + ] + builds_and_revisions_list = [(47483, 1), (47483, 1)] + self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list) + def test_fetch_build(self): buildbot = BuildBot() builder = Builder(u"Test Builder \u2661", buildbot) @@ -137,8 +155,8 @@ class BuildTest(unittest.TestCase): def test_layout_test_results(self): buildbot = BuildBot() builder = Builder(u"Foo Builder (test)", buildbot) + builder._fetch_file_from_results = lambda results_url, file_name: None build = Build(builder, None, None, None) - build._fetch_file_from_results = lambda file_name: None # Test that layout_test_results() returns None if the fetch fails. self.assertEqual(build.layout_test_results(), None) diff --git a/Tools/Scripts/webkitpy/common/system/executive_mock.py b/Tools/Scripts/webkitpy/common/system/executive_mock.py index a15c3654d..cfe989968 100644 --- a/Tools/Scripts/webkitpy/common/system/executive_mock.py +++ b/Tools/Scripts/webkitpy/common/system/executive_mock.py @@ -88,7 +88,10 @@ class MockExecutive(object): env_string = "" if env: env_string = ", env=%s" % env - log("MOCK run_command: %s, cwd=%s%s" % (args, cwd, env_string)) + input_string = "" + if input: + input_string = ", input=%s" % input + log("MOCK run_command: %s, cwd=%s%s%s" % (args, cwd, env_string, input_string)) output = "MOCK output of child process" if self._should_throw: raise ScriptError("MOCK ScriptError", output=output) diff --git a/Tools/Scripts/webkitpy/common/system/user.py b/Tools/Scripts/webkitpy/common/system/user.py index e20405912..262b97944 100644 --- a/Tools/Scripts/webkitpy/common/system/user.py +++ b/Tools/Scripts/webkitpy/common/system/user.py @@ -74,14 +74,20 @@ class User(object): return cls.prompt(message, repeat=repeat, raw_input=getpass.getpass) @classmethod - def prompt_with_list(cls, list_title, list_items, can_choose_multiple=False, raw_input=raw_input): + def prompt_with_multiple_lists(cls, list_title, subtitles, lists, can_choose_multiple=False, raw_input=raw_input): + item_index = 0 + cumulated_list = [] print list_title - i = 0 - for item in list_items: - i += 1 - print "%2d. %s" % (i, item) + for i in range(len(subtitles)): + print "\n" + subtitles[i] + for item in lists[i]: + item_index += 1 + print "%2d. %s" % (item_index, item) + cumulated_list += lists[i] + return cls._wait_on_list_response(cumulated_list, can_choose_multiple, raw_input) - # Loop until we get valid input + @classmethod + def _wait_on_list_response(cls, list_items, can_choose_multiple, raw_input): while True: if can_choose_multiple: response = cls.prompt("Enter one or more numbers (comma-separated), or \"all\": ", raw_input=raw_input) @@ -99,6 +105,15 @@ class User(object): continue return list_items[result] + @classmethod + def prompt_with_list(cls, list_title, list_items, can_choose_multiple=False, raw_input=raw_input): + print list_title + i = 0 + for item in list_items: + i += 1 + print "%2d. %s" % (i, item) + return cls._wait_on_list_response(list_items, can_choose_multiple, raw_input) + def edit(self, files): editor = os.environ.get("EDITOR") or "vi" args = shlex.split(editor) diff --git a/Tools/Scripts/webkitpy/common/system/user_unittest.py b/Tools/Scripts/webkitpy/common/system/user_unittest.py index 7ec9b34e4..8b7cc1c0c 100644 --- a/Tools/Scripts/webkitpy/common/system/user_unittest.py +++ b/Tools/Scripts/webkitpy/common/system/user_unittest.py @@ -51,6 +51,32 @@ class UserTest(unittest.TestCase): return None self.assertEqual(User.prompt("input", repeat=self.repeatsRemaining, raw_input=mock_raw_input), None) + def test_prompt_with_multiple_lists(self): + def run_prompt_test(inputs, expected_result, can_choose_multiple=False): + def mock_raw_input(message): + return inputs.pop(0) + output_capture = OutputCapture() + actual_result = output_capture.assert_outputs( + self, + User.prompt_with_multiple_lists, + args=["title", ["subtitle1", "subtitle2"], [["foo", "bar"], ["foobar", "barbaz"]]], + kwargs={"can_choose_multiple": can_choose_multiple, "raw_input": mock_raw_input}, + expected_stdout="title\n\nsubtitle1\n 1. foo\n 2. bar\n\nsubtitle2\n 3. foobar\n 4. barbaz\n") + self.assertEqual(actual_result, expected_result) + self.assertEqual(len(inputs), 0) + + run_prompt_test(["1"], "foo") + run_prompt_test(["badinput", "2"], "bar") + run_prompt_test(["3"], "foobar") + run_prompt_test(["4"], "barbaz") + + run_prompt_test(["1,2"], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test([" 1, 2 "], ["foo", "bar"], can_choose_multiple=True) + run_prompt_test(["all"], ["foo", "bar", 'foobar', 'barbaz'], can_choose_multiple=True) + run_prompt_test([""], ["foo", "bar", 'foobar', 'barbaz'], can_choose_multiple=True) + run_prompt_test([" "], ["foo", "bar", 'foobar', 'barbaz'], can_choose_multiple=True) + run_prompt_test(["badinput", "all"], ["foo", "bar", 'foobar', 'barbaz'], can_choose_multiple=True) + def test_prompt_with_list(self): def run_prompt_test(inputs, expected_result, can_choose_multiple=False): def mock_raw_input(message): diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py index a63c1efb8..f2fed3f4a 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py @@ -266,9 +266,7 @@ class TestRunInterruptedException(Exception): return self.__class__, (self.reason,) -class WorkerException(Exception): - """Raised when we receive an unexpected/unknown exception from a worker.""" - pass +WorkerException = manager_worker_broker.WorkerException class TestShard(object): @@ -771,25 +769,17 @@ class Manager(object): num_workers = min(num_workers, len(all_shards)) self._log_num_workers(num_workers, len(all_shards), len(locked_shards)) - manager_connection = manager_worker_broker.get(num_workers, self, worker.Worker) + def worker_factory(worker_connection, worker_number): + return worker.Worker(worker_connection, worker_number, self.results_directory(), self._options) + + manager_connection = manager_worker_broker.get(num_workers, self, worker_factory, self._port.host) if self._options.dry_run: return (keyboard_interrupted, interrupted, thread_timings, self._group_stats, self._all_results) self._printer.print_update('Starting %s ...' % grammar.pluralize('worker', num_workers)) for worker_number in xrange(num_workers): - worker_arguments = worker.WorkerArguments(worker_number, self.results_directory(), self._options) - worker_connection = manager_connection.start_worker(worker_arguments) - if num_workers == 1: - # FIXME: We need to be able to share a port with the work so - # that some of the tests can query state on the port; ideally - # we'd rewrite the tests so that this wasn't necessary. - # - # Note that this only works because in the inline case - # the worker hasn't really started yet and won't start - # running until we call run_message_loop(), below. - worker_connection.set_inline_arguments(self._port) - + worker_connection = manager_connection.start_worker(worker_number) worker_state = _WorkerState(worker_number, worker_connection) self._worker_states[worker_connection.name()] = worker_state @@ -1515,7 +1505,7 @@ class Manager(object): def _log_messages(self, messages): for message in messages: - self._printer.writeln(*message) + logging.root.handle(message) def _log_worker_stack(self, stack): webkitpydir = self._port.path_from_webkit_base('Tools', 'Scripts', 'webkitpy') + self._filesystem.sep diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py index 7047d939c..f7baced0a 100755 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py @@ -69,12 +69,15 @@ import cPickle import logging import multiprocessing import optparse +import os import Queue import sys import traceback +from webkitpy.common.host import Host from webkitpy.common.system import stack_utils +from webkitpy.layout_tests.views import metered_stream _log = logging.getLogger(__name__) @@ -87,15 +90,15 @@ MANAGER_TOPIC = 'managers' ANY_WORKER_TOPIC = 'workers' -def get(max_workers, client, worker_class): +def get(max_workers, client, worker_factory, host=None): """Return a connection to a manager/worker message_broker Args: max_workers - max # of workers to run concurrently. client - BrokerClient implementation to dispatch replies to. - worker_class - type of workers to create. This class should override - the methods in AbstractWorker. + worker_factory: factory method for creating objects that implement the Worker interface. + host: optional picklable host object that can be passed to workers for testing. Returns: A handle to an object that will talk to a message broker configured for the normal manager/worker communication.""" @@ -107,7 +110,12 @@ def get(max_workers, client, worker_class): manager_class = _MultiProcessManager broker = _Broker(queue_class) - return manager_class(broker, client, worker_class) + return manager_class(broker, client, worker_factory, host) + + +class WorkerException(Exception): + """Raised when we receive an unexpected/unknown exception from a worker.""" + pass class BrokerClient(object): @@ -247,30 +255,25 @@ class _BrokerConnection(object): message_name, *message_args) def raise_exception(self, exc_info): - # Since tracebacks aren't picklable, send the extracted stack instead. + # Since tracebacks aren't picklable, send the extracted stack instead, + # but at least log the full traceback. exception_type, exception_value, exception_traceback = sys.exc_info() - stack_utils.log_traceback(_log.debug, exception_traceback) + stack_utils.log_traceback(_log.error, exception_traceback) stack = traceback.extract_tb(exception_traceback) self._broker.post_message(self._client, self._post_topic, 'exception', exception_type, exception_value, stack) class AbstractWorker(BrokerClient): - def __init__(self, worker_connection, worker_arguments=None): - """The constructor should be used to do any simple initialization - necessary, but should not do anything that creates data structures - that cannot be Pickled or sent across processes (like opening - files or sockets). Complex initialization should be done at the - start of the run() call. - - Args: - worker_connection - handle to the _BrokerConnection object creating - the worker and that can be used for messaging. - worker_arguments - (optional, Picklable) object passed to the worker from the manager""" + def __init__(self, worker_connection, worker_number): BrokerClient.__init__(self) + self.worker = None self._worker_connection = worker_connection - self._name = 'worker' + self._worker_number = worker_number + self._name = 'worker/%d' % worker_number self._done = False self._canceled = False + self._options = optparse.Values({'verbose': False}) + self.host = None def name(self): return self._name @@ -281,10 +284,14 @@ class AbstractWorker(BrokerClient): def stop_handling_messages(self): self._done = True - def run(self): + def run(self, host): """Callback for the worker to start executing. Typically does any remaining initialization and then calls broker_connection.run_message_loop().""" exception_msg = "" + self.host = host + + self.worker.safe_init() + _log.debug('%s starting' % self._name) try: self._worker_connection.run_message_loop() @@ -299,6 +306,18 @@ class AbstractWorker(BrokerClient): self._worker_connection.raise_exception(sys.exc_info()) finally: _log.debug("%s done with message loop%s" % (self._name, exception_msg)) + try: + self.worker.cleanup() + finally: + # Make sure we post a done so that we can flush the log messages + # and clean up properly even if we raise an exception in worker.cleanup(). + self._worker_connection.post_message('done') + + def handle_stop(self, source): + self._done = True + + def handle_test_list(self, source, list_name, test_list): + self.worker.handle('test_list', source, list_name, test_list) def cancel(self): """Called when possible to indicate to the worker to stop processing @@ -306,44 +325,33 @@ class AbstractWorker(BrokerClient): method being called, so clients should not rely solely on this.""" self._canceled = True + def yield_to_broker(self): + self._worker_connection.yield_to_broker() -class _ManagerConnection(_BrokerConnection): - def __init__(self, broker, client, worker_class): - """Base initialization for all Manager objects. + def post_message(self, *args): + self._worker_connection.post_message(*args) - Args: - broker: handle to the message_broker object - client: callback object (the caller) - worker_class: class object to use to create workers. - """ - _BrokerConnection.__init__(self, broker, client, MANAGER_TOPIC, ANY_WORKER_TOPIC) - self._worker_class = worker_class - def start_worker(self, worker_arguments=None): - """Starts a new worker. +class _ManagerConnection(_BrokerConnection): + def __init__(self, broker, client, worker_factory, host): + _BrokerConnection.__init__(self, broker, client, MANAGER_TOPIC, ANY_WORKER_TOPIC) + self._worker_factory = worker_factory + self._host = host - Args: - worker_arguments - an optional Picklable object that is passed to the worker constructor - """ + def start_worker(self, worker_number): raise NotImplementedError class _InlineManager(_ManagerConnection): - def __init__(self, broker, client, worker_class): - _ManagerConnection.__init__(self, broker, client, worker_class) + def __init__(self, broker, client, worker_factory, host): + _ManagerConnection.__init__(self, broker, client, worker_factory, host) self._inline_worker = None - def start_worker(self, worker_arguments=None): - self._inline_worker = _InlineWorkerConnection(self._broker, - self._client, self._worker_class, worker_arguments) + def start_worker(self, worker_number): + host = self._host + self._inline_worker = _InlineWorkerConnection(host, self._broker, self._client, self._worker_factory, worker_number) return self._inline_worker - def set_inline_arguments(self, arguments=None): - # Note that this method only exists here, and not on all - # ManagerConnections; calling this method on a MultiProcessManager - # will deliberately result in a runtime error. - self._inline_worker.set_inline_arguments(arguments) - def run_message_loop(self, delay_secs=None): # Note that delay_secs is ignored in this case since we can't easily # implement it. @@ -352,16 +360,34 @@ class _InlineManager(_ManagerConnection): class _MultiProcessManager(_ManagerConnection): - def start_worker(self, worker_arguments=None): - worker_connection = _MultiProcessWorkerConnection(self._broker, - self._worker_class, worker_arguments) + def _can_pickle_host(self): + try: + cPickle.dumps(self._host) + return True + except TypeError: + return False + + def start_worker(self, worker_number): + host = None + if self._can_pickle_host(): + host = self._host + worker_connection = _MultiProcessWorkerConnection(host, self._broker, self._worker_factory, worker_number) worker_connection.start() return worker_connection class _WorkerConnection(_BrokerConnection): - def __init__(self, broker, worker_class, worker_arguments=None): - self._client = worker_class(self, worker_arguments) + def __init__(self, host, broker, worker_factory, worker_number): + # FIXME: keeping track of the differences between the WorkerConnection, the AbstractWorker, and the + # actual Worker (created by worker_factory) is very confusing, but this all gets better when + # _WorkerConnection and AbstractWorker get merged. + self._client = AbstractWorker(self, worker_number) + self._worker = worker_factory(self._client, worker_number) + self._client.worker = self._worker + self._host = host + self._log_messages = [] + self._logger = None + self._log_handler = None _BrokerConnection.__init__(self, broker, self._client, ANY_WORKER_TOPIC, MANAGER_TOPIC) def name(self): @@ -379,10 +405,37 @@ class _WorkerConnection(_BrokerConnection): def yield_to_broker(self): pass + def post_message(self, *args): + # FIXME: This is a hack until we can remove the log_messages arg from the manager. + if args[0] in ('finished_test', 'done'): + log_messages = self._log_messages + self._log_messages = [] + args = args + tuple([log_messages]) + super(_WorkerConnection, self).post_message(*args) + + def set_up_logging(self): + self._logger = logging.root + # The unix multiprocessing implementation clones the MeteredStream log handler + # into the child process, so we need to remove it to avoid duplicate logging. + for h in self._logger.handlers: + # log handlers don't have names until python 2.7. + if getattr(h, 'name', '') == metered_stream.LOG_HANDLER_NAME: + self._logger.removeHandler(h) + break + self._logger.setLevel(logging.DEBUG if self._client._options.verbose else logging.INFO) + self._log_handler = _WorkerLogHandler(self) + self._logger.addHandler(self._log_handler) + + def clean_up_logging(self): + if self._log_handler and self._logger: + self._logger.removeHandler(self._log_handler) + self._log_handler = None + self._logger = None + class _InlineWorkerConnection(_WorkerConnection): - def __init__(self, broker, manager_client, worker_class, worker_arguments): - _WorkerConnection.__init__(self, broker, worker_class, worker_arguments) + def __init__(self, host, broker, manager_client, worker_factory, worker_number): + _WorkerConnection.__init__(self, host, broker, worker_factory, worker_number) self._alive = False self._manager_client = manager_client @@ -395,13 +448,10 @@ class _InlineWorkerConnection(_WorkerConnection): def join(self, timeout): assert not self._alive - def set_inline_arguments(self, arguments): - self._client.set_inline_arguments(arguments) - def run(self): self._alive = True try: - self._client.run() + self._client.run(self._host) finally: self._alive = False @@ -411,8 +461,11 @@ class _InlineWorkerConnection(_WorkerConnection): def raise_exception(self, exc_info): # Since the worker is in the same process as the manager, we can # raise the exception directly, rather than having to send it through - # the queue. This allows us to preserve the traceback. - raise exc_info[0], exc_info[1], exc_info[2] + # the queue. This allows us to preserve the traceback, but we log + # it anyway for consistency with the multiprocess case. + exception_type, exception_value, exception_traceback = sys.exc_info() + stack_utils.log_traceback(_log.error, exception_traceback) + raise exception_type, exception_value, exception_traceback class _Process(multiprocessing.Process): @@ -422,12 +475,14 @@ class _Process(multiprocessing.Process): self._client = client def run(self): - self._client.run() + if not self._worker_connection._host: + self._worker_connection._host = Host() + self._worker_connection.run() class _MultiProcessWorkerConnection(_WorkerConnection): - def __init__(self, broker, worker_class, worker_arguments): - _WorkerConnection.__init__(self, broker, worker_class, worker_arguments) + def __init__(self, host, broker, worker_factory, worker_number): + _WorkerConnection.__init__(self, host, broker, worker_factory, worker_number) self._proc = _Process(self, self._client) def cancel(self): @@ -441,3 +496,20 @@ class _MultiProcessWorkerConnection(_WorkerConnection): def start(self): self._proc.start() + + def run(self): + self.set_up_logging() + try: + self._client.run(self._host) + finally: + self.clean_up_logging() + + +class _WorkerLogHandler(logging.Handler): + def __init__(self, worker): + logging.Handler.__init__(self) + self._worker = worker + self._pid = os.getpid() + + def emit(self, record): + self._worker._log_messages.append(record) diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py index 046425664..d7c3714d8 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py @@ -44,8 +44,7 @@ starting_queue = None stopping_queue = None -WORKER_NAME = 'TestWorker' - +WORKER_NAME = 'worker/1' def make_broker(manager, max_workers, start_queue=None, stop_queue=None): global starting_queue @@ -55,35 +54,34 @@ def make_broker(manager, max_workers, start_queue=None, stop_queue=None): return manager_worker_broker.get(max_workers, manager, _TestWorker) -class _TestWorker(manager_worker_broker.AbstractWorker): - def __init__(self, worker_connection, worker_arguments=None): - super(_TestWorker, self).__init__(worker_connection) - self._name = WORKER_NAME +class _TestWorker(object): + def __init__(self, caller, worker_number): + self._caller = caller self._thing_to_greet = 'everybody' self._starting_queue = starting_queue self._stopping_queue = stopping_queue + self._options = optparse.Values({'verbose': False}) - def set_inline_arguments(self, thing_to_greet): - self._thing_to_greet = thing_to_greet + def name(self): + return WORKER_NAME - def handle_stop(self, src): - self.stop_handling_messages() + def cleanup(self): + pass - def handle_test(self, src, an_int, a_str): + def handle(self, message, src, an_int, a_str): assert an_int == 1 assert a_str == "hello, world" - self._worker_connection.post_message('test', 2, 'hi, ' + self._thing_to_greet) + self._caller.post_message('finished_test', 2) - def run(self): + def safe_init(self): if self._starting_queue: self._starting_queue.put('') if self._stopping_queue: self._stopping_queue.get() - try: - super(_TestWorker, self).run() - finally: - self._worker_connection.post_message('done') + + def stop(self): + self._caller.post_message('done') class FunctionTests(unittest.TestCase): @@ -107,19 +105,17 @@ class _TestsMixin(object): def is_done(self): return self._done - def handle_done(self, src): + def handle_done(self, src, log_messages): self._done = True - def handle_test(self, src, an_int, a_str): + def handle_finished_test(self, src, an_int, log_messages): self._an_int = an_int - self._a_str = a_str def handle_exception(self, src, exception_type, exception_value, stack): raise exception_type(exception_value) def setUp(self): self._an_int = None - self._a_str = None self._broker = None self._done = False self._exception = None @@ -131,7 +127,7 @@ class _TestsMixin(object): def test_name(self): self.make_broker() - worker = self._broker.start_worker() + worker = self._broker.start_worker(1) self.assertEquals(worker.name(), WORKER_NAME) worker.cancel() worker.join(0.1) @@ -140,8 +136,8 @@ class _TestsMixin(object): def test_cancel(self): self.make_broker() - worker = self._broker.start_worker() - self._broker.post_message('test', 1, 'hello, world') + worker = self._broker.start_worker(1) + self._broker.post_message('test_list', 1, 'hello, world') worker.cancel() worker.join(0.1) self.assertFalse(worker.is_alive()) @@ -149,20 +145,19 @@ class _TestsMixin(object): def test_done(self): self.make_broker() - worker = self._broker.start_worker() - self._broker.post_message('test', 1, 'hello, world') + worker = self._broker.start_worker(1) + self._broker.post_message('test_list', 1, 'hello, world') self._broker.post_message('stop') self._broker.run_message_loop() worker.join(0.5) self.assertFalse(worker.is_alive()) self.assertTrue(self.is_done()) self.assertEqual(self._an_int, 2) - self.assertEqual(self._a_str, 'hi, everybody') self._broker.cleanup() def test_unknown_message(self): self.make_broker() - worker = self._broker.start_worker() + worker = self._broker.start_worker(1) self._broker.post_message('unknown') try: self._broker.run_message_loop() @@ -181,15 +176,6 @@ class InlineBrokerTests(_TestsMixin, unittest.TestCase): _TestsMixin.setUp(self) self._max_workers = 1 - def test_inline_arguments(self): - self.make_broker() - worker = self._broker.start_worker() - worker.set_inline_arguments('me') - self._broker.post_message('test', 1, 'hello, world') - self._broker.post_message('stop') - self._broker.run_message_loop() - self.assertEquals(self._a_str, 'hi, me') - # FIXME: https://bugs.webkit.org/show_bug.cgi?id=54520. if sys.platform not in ('cygwin', 'win32'): @@ -200,35 +186,6 @@ if sys.platform not in ('cygwin', 'win32'): self._max_workers = 2 -class InterfaceTest(unittest.TestCase): - # These tests mostly exist to pacify coverage. - - # FIXME: There must be a better way to do this and also verify - # that classes do implement every abstract method in an interface. - def test_brokerclient_is_abstract(self): - # Test that all the base class methods are abstract and have the - # signature we expect. - obj = manager_worker_broker.BrokerClient() - self.assertRaises(NotImplementedError, obj.is_done) - self.assertRaises(NotImplementedError, obj.name) - - def test_managerconnection_is_abstract(self): - # Test that all the base class methods are abstract and have the - # signature we expect. - broker = make_broker(self, 1) - obj = manager_worker_broker._ManagerConnection(broker._broker, self, None) - self.assertRaises(NotImplementedError, obj.start_worker) - - def test_workerconnection_is_abstract(self): - # Test that all the base class methods are abstract and have the - # signature we expect. - broker = make_broker(self, 1) - obj = manager_worker_broker._WorkerConnection(broker._broker, _TestWorker, None) - self.assertRaises(NotImplementedError, obj.cancel) - self.assertRaises(NotImplementedError, obj.is_alive) - self.assertRaises(NotImplementedError, obj.join, None) - - class MessageTest(unittest.TestCase): def test__no_body(self): msg = manager_worker_broker._Message('src', 'topic_name', 'message_name', None) diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py b/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py index d321b5a41..c68915916 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py @@ -29,36 +29,27 @@ """Handle messages from the Manager and executes actual tests.""" import logging -import os -import sys import threading import time -from webkitpy.common.host import Host -from webkitpy.layout_tests.controllers import manager_worker_broker from webkitpy.layout_tests.controllers import single_test_runner from webkitpy.layout_tests.models import test_expectations from webkitpy.layout_tests.models import test_results -from webkitpy.layout_tests.views import metered_stream _log = logging.getLogger(__name__) -class WorkerArguments(object): - def __init__(self, worker_number, results_directory, options): - self.worker_number = worker_number - self.results_directory = results_directory - self.options = options +class Worker(object): + def __init__(self, worker_connection, worker_number, results_directory, options): + self._worker_connection = worker_connection + self._worker_number = worker_number + self._name = 'worker/%d' % worker_number + self._results_directory = results_directory + self._options = options - -class Worker(manager_worker_broker.AbstractWorker): - def __init__(self, worker_connection, worker_arguments): - super(Worker, self).__init__(worker_connection, worker_arguments) - self._worker_number = worker_arguments.worker_number - self._name = 'worker/%d' % self._worker_number - self._results_directory = worker_arguments.results_directory - self._options = worker_arguments.options + # The remaining fields are initialized in safe_init() + self._host = None self._port = None self._batch_size = None self._batch_count = None @@ -66,70 +57,24 @@ class Worker(manager_worker_broker.AbstractWorker): self._driver = None self._tests_run_file = None self._tests_run_filename = None - self._log_messages = [] - self._logger = None - self._log_handler = None def __del__(self): self.cleanup() def safe_init(self): - """This method should only be called when it is is safe for the mixin - to create state that can't be Pickled. + """This method is called when it is safe for the object to create state that + does not need to be pickled (usually this means it is called in a child process).""" + self._host = self._worker_connection.host + self._filesystem = self._host.filesystem + self._port = self._host.port_factory.get(self._options.platform, self._options) - This routine exists so that the mixin can be created and then marshaled - across into a child process.""" - self._filesystem = self._port.host.filesystem self._batch_count = 0 self._batch_size = self._options.batch_size or 0 tests_run_filename = self._filesystem.join(self._results_directory, "tests_run%d.txt" % self._worker_number) self._tests_run_file = self._filesystem.open_text_file_for_writing(tests_run_filename) - def _set_up_logging(self): - self._logger = logging.getLogger() - - # The unix multiprocessing implementation clones the MeteredStream log handler - # into the child process, so we need to remove it to avoid duplicate logging. - for h in self._logger.handlers: - # log handlers don't have names until python 2.7. - if getattr(h, 'name', '') == metered_stream.LOG_HANDLER_NAME: - self._logger.removeHandler(h) - break - - self._logger.setLevel(logging.DEBUG if self._options.verbose else logging.INFO) - self._log_handler = _WorkerLogHandler(self) - self._logger.addHandler(self._log_handler) - - def _set_up_host_and_port(self): - options = self._options - if options.platform and 'test' in options.platform: - # It is lame to import mocks into real code, but this allows us to use the test port in multi-process tests as well. - from webkitpy.common.host_mock import MockHost - host = MockHost() - else: - host = Host() - self._port = host.port_factory.get(options.platform, options) - - def set_inline_arguments(self, port): - self._port = port - - def run(self): - if not self._port: - # We are running in a child process and need to initialize things. - self._set_up_logging() - self._set_up_host_and_port() - - self.safe_init() - try: - _log.debug("%s starting" % self._name) - super(Worker, self).run() - finally: - self.kill_driver() - _log.debug("%s exiting" % self._name) - self.cleanup() - self._worker_connection.post_message('done', self._log_messages) - - def handle_test_list(self, src, list_name, test_list): + def handle(self, name, source, list_name, test_list): + assert name == 'test_list' start_time = time.time() num_tests = 0 for test_input in test_list: @@ -141,9 +86,6 @@ class Worker(manager_worker_broker.AbstractWorker): elapsed_time = time.time() - start_time self._worker_connection.post_message('finished_list', list_name, num_tests, elapsed_time) - def handle_stop(self, src): - self.stop_handling_messages() - def _update_test_input(self, test_input): test_input.reference_files = self._port.reference_files(test_input.test_name) if test_input.reference_files: @@ -164,9 +106,7 @@ class Worker(manager_worker_broker.AbstractWorker): result = self.run_test_with_timeout(test_input, test_timeout_sec) elapsed_time = time.time() - start - log_messages = self._log_messages - self._log_messages = [] - self._worker_connection.post_message('finished_test', result, elapsed_time, log_messages) + self._worker_connection.post_message('finished_test', result, elapsed_time) self.clean_up_after_test(test_input, result) @@ -176,10 +116,6 @@ class Worker(manager_worker_broker.AbstractWorker): if self._tests_run_file: self._tests_run_file.close() self._tests_run_file = None - if self._log_handler and self._logger: - self._logger.removeHandler(self._log_handler) - self._log_handler = None - self._logger = None def timeout(self, test_input): """Compute the appropriate timeout value for a test.""" @@ -198,10 +134,13 @@ class Worker(manager_worker_broker.AbstractWorker): return thread_timeout_sec def kill_driver(self): - if self._driver: + # Be careful about how and when we kill the driver; if driver.stop() + # raises an exception, this routine may get re-entered via __del__. + driver = self._driver + self._driver = None + if driver: _log.debug("%s killing driver" % self._name) - self._driver.stop() - self._driver = None + driver.stop() def run_test_with_timeout(self, test_input, timeout): if self._options.run_singly: @@ -296,13 +235,3 @@ class Worker(manager_worker_broker.AbstractWorker): def run_single_test(self, driver, test_input): return single_test_runner.run_single_test(self._port, self._options, test_input, driver, self._name) - - -class _WorkerLogHandler(logging.Handler): - def __init__(self, worker): - logging.Handler.__init__(self) - self._worker = worker - self._pid = os.getpid() - - def emit(self, record): - self._worker._log_messages.append(tuple([record.getMessage(), record.created, self._pid])) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/apple.py b/Tools/Scripts/webkitpy/layout_tests/port/apple.py index 8807d40d7..5e8b8b829 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/apple.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/apple.py @@ -51,11 +51,16 @@ class ApplePort(WebKitPort): @classmethod def determine_full_port_name(cls, host, options, port_name): + # If the port_name matches the (badly named) cls.port_name, that + # means that they passed 'mac' or 'win' and didn't specify a version. + # That convention means that we're supposed to use the version currently + # being run, so this won't work if you're not on mac or win (respectively). + # If you're not on the o/s in question, you must specify a full version or -future (cf. above). if port_name == cls.port_name: assert port_name == host.platform.os_name return cls.port_name + '-' + host.platform.os_version if port_name == cls.port_name + '-wk2': - assert port_name == host.platform.os_name + assert port_name == host.platform.os_name + '-wk2' return cls.port_name + '-' + host.platform.os_version + '-wk2' return port_name diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py index 0fa7e3f3f..fbf0b930b 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py @@ -136,8 +136,12 @@ class Port(object): self._reftest_list = {} self._results_directory = None - def default_test_timeout_ms(self): - return 6 * 1000 + def default_pixel_tests(self): + # FIXME: Disable until they are run by default on build.webkit.org. + return False + + def default_timeout_ms(self): + return 35 * 1000 def wdiff_available(self): if self._wdiff_available is None: @@ -650,6 +654,9 @@ class Port(object): return True return False + def is_chromium(self): + return False + def name(self): """Returns a name that uniquely identifies this particular type of port (e.g., "mac-snowleopard" or "chromium-linux-x86_x64" and can be passed @@ -693,7 +700,7 @@ class Port(object): # test_expectations are always in mac/ not mac-leopard/ by convention, hence we use port_name instead of name(). port_name = self.port_name - if port_name.startswith('chromium') or port_name.startswith('google-chrome'): + if port_name.startswith('chromium'): port_name = 'chromium' return self._filesystem.join(self._webkit_baseline_path(port_name), 'TestExpectations') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/builders.py b/Tools/Scripts/webkitpy/layout_tests/port/builders.py index 3ad45be17..21fcc9801 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/builders.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/builders.py @@ -44,25 +44,31 @@ _exact_matches = { "Webkit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])}, "Webkit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])}, "Webkit Linux (dbg)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])}, - "Webkit Mac10.5": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard"])}, - "Webkit Mac10.5 (dbg)(1)": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard", "debug"])}, - "Webkit Mac10.5 (dbg)(2)": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard", "debug"])}, "Webkit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])}, "Webkit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])}, "Webkit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, # These builders are on build.webkit.org. - "Apple Lion Release WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion"])}, - "Apple Lion Debug WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "debug"])}, + # FIXME: Remove rebaseline_override_dir once there is an Apple buildbot that corresponds to platform/mac (i.e. a Mountain Lion bot). + "Apple Lion Release WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion"]), "rebaseline_override_dir": "mac"}, + "Apple Lion Debug WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "debug"]), "rebaseline_override_dir": "mac"}, "Apple Lion Release WK2 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "wk2"])}, "Apple Lion Debug WK2 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "wk2", "debug"])}, "Apple Win XP Debug (Tests)": {"port_name": "win-xp", "specifiers": set(["win", "debug"])}, - "Apple Win 7 Release (Tests)": {"port_name": "win-xp", "specifiers": set(["win"])}, + # FIXME: Remove rebaseline_override_dir once there is an Apple buildbot that corresponds to platform/win. + "Apple Win 7 Release (Tests)": {"port_name": "win-7sp0", "specifiers": set(["win"]), "rebaseline_override_dir": "win"}, - "GTK Linux 32-bit Debug": {"port_name": "gtk", "specifiers": set(["gtk"])}, - "Qt Linux Release": {"port_name": "qt-linux", "specifiers": set(["win", "linux", "mac"])}, - "EFL Linux Release": {"port_name": "efl", "specifiers": set(["efl"])}, + "GTK Linux 32-bit Release": {"port_name": "gtk", "specifiers": set(["gtk", "x86", "release"])}, + "GTK Linux 64-bit Debug": {"port_name": "gtk", "specifiers": set(["gtk", "x86_64", "debug"])}, + "GTK Linux 64-bit Release": {"port_name": "gtk", "specifiers": set(["gtk", "x86_64", "release"])}, + "GTK Linux 64-bit Release WK2 (Tests)": {"port_name": "gtk", "specifiers": set(["gtk", "x86_64", "wk2", "release"])}, + + # FIXME: Remove rebaseline_override_dir once there are Qt bots for all the platform/qt-* directories. + "Qt Linux Release": {"port_name": "qt-linux", "specifiers": set(["win", "linux", "mac"]), "rebaseline_override_dir": "qt"}, + + "EFL Linux 64-bit Debug": {"port_name": "efl", "specifiers": set(["efl", "debug"])}, + "EFL Linux 64-bit Release": {"port_name": "efl", "specifiers": set(["efl", "release"])}, } @@ -70,7 +76,6 @@ _fuzzy_matches = { # These builders are on build.webkit.org. r"SnowLeopard": "mac-snowleopard", r"Apple Lion": "mac-lion", - r"Leopard": "mac-leopard", r"Windows": "win", r"GTK": "gtk", r"Qt": "qt", @@ -81,8 +86,6 @@ _fuzzy_matches = { _ports_without_builders = [ - "google-chrome-linux32", - "google-chrome-linux64", "qt-mac", "qt-win", "qt-wk2", @@ -93,12 +96,10 @@ def builder_path_from_name(builder_name): return re.sub(r'[\s().]', '_', builder_name) -@memoized def all_builder_names(): return sorted(set(_exact_matches.keys())) -@memoized def all_port_names(): return sorted(set(map(lambda x: x["port_name"], _exact_matches.values()) + _ports_without_builders)) @@ -107,6 +108,10 @@ def coverage_specifiers_for_builder_name(builder_name): return _exact_matches[builder_name].get("specifiers", set()) +def rebaseline_override_dir(builder_name): + return _exact_matches[builder_name].get("rebaseline_override_dir", None) + + def port_name_for_builder_name(builder_name): if builder_name in _exact_matches: return _exact_matches[builder_name]["port_name"] diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index 5aad94c0f..24f7efa0f 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -39,20 +39,20 @@ import sys import time from webkitpy.common.system import executive +from webkitpy.common.system.path import cygpath from webkitpy.layout_tests.models.test_configuration import TestConfiguration from webkitpy.layout_tests.port.base import Port, VirtualTestSuite from webkitpy.layout_tests.port.driver import DriverOutput -from webkitpy.layout_tests.port.webkit import WebKitDriver +from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver _log = logging.getLogger(__name__) -class ChromiumPort(Port): +class ChromiumPort(WebKitPort): """Abstract base class for Chromium implementations of the Port class.""" ALL_SYSTEMS = ( - ('leopard', 'x86'), ('snowleopard', 'x86'), ('lion', 'x86'), ('xp', 'x86'), @@ -70,7 +70,7 @@ class ChromiumPort(Port): ] CONFIGURATION_SPECIFIER_MACROS = { - 'mac': ['leopard', 'snowleopard', 'lion'], + 'mac': ['snowleopard', 'lion'], 'win': ['xp', 'win7'], 'linux': ['lucid'], 'android': ['icecreamsandwich'], @@ -106,10 +106,21 @@ class ChromiumPort(Port): return module_path[0:offset] def __init__(self, host, port_name, **kwargs): - Port.__init__(self, host, port_name, **kwargs) + super(ChromiumPort, self).__init__(host, port_name, **kwargs) # All sub-classes override this, but we need an initial value for testing. self._chromium_base_dir_path = None + def is_chromium(self): + return True + + def default_pixel_tests(self): + return True + + def default_timeout_ms(self): + if self.get_option('configuration') == 'Debug': + return 12 * 1000 + return 6 * 1000 + def _check_file_exists(self, path_to_file, file_description, override_step=None, logging=True): """Verify the file is present where expected or log an error. @@ -129,6 +140,11 @@ class ChromiumPort(Port): return False return True + def driver_name(self): + # FIXME: merge this with Port.driver_name, WebKitPort.driver_name + if self.get_option('driver_name'): + return self.get_option('driver_name') + return 'DumpRenderTree' def check_build(self, needs_http): result = True @@ -256,6 +272,20 @@ class ChromiumPort(Port): except AssertionError: return self._build_path(self.get_option('configuration'), 'layout-test-results') + def _driver_class(self): + return ChromiumDriver + + def _missing_symbol_to_skipped_tests(self): + # FIXME: Should WebKitPort have these definitions also? + return { + "ff_mp3_decoder": ["webaudio/codec-tests/mp3"], + "ff_aac_decoder": ["webaudio/codec-tests/aac"], + } + + def skipped_layout_tests(self, test_list): + # FIXME: Merge w/ WebKitPort.skipped_layout_tests() + return set(self._skipped_tests_for_unsupported_features(test_list)) + def setup_test_run(self): # Delete the disk cache if any to ensure a clean test run. dump_render_tree_binary_path = self._path_to_driver() @@ -264,9 +294,6 @@ class ChromiumPort(Port): if self._filesystem.exists(cachedir): self._filesystem.rmtree(cachedir) - def _driver_class(self): - return ChromiumDriver - def start_helper(self): helper_path = self._path_to_helper() if helper_path: @@ -321,7 +348,13 @@ class ChromiumPort(Port): ]) def expectations_files(self): - paths = [self.path_to_test_expectations_file(), self.path_from_chromium_base('skia', 'skia_test_expectations.txt')] + paths = [self.path_to_test_expectations_file()] + skia_expectations_path = self.path_from_chromium_base('skia', 'skia_test_expectations.txt') + if self._filesystem.exists(skia_expectations_path): + paths.append(skia_expectations_path) + else: + _log.warning("Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky.") + builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME') if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names: paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt')) @@ -365,6 +398,10 @@ class ChromiumPort(Port): def _build_path(self, *comps): return self._static_build_path(self._filesystem, self.get_option('build_directory'), self.path_from_chromium_base(), self.path_from_webkit_base(), *comps) + def _path_to_image_diff(self): + binary_name = 'ImageDiff' + return self._build_path(self.get_option('configuration'), binary_name) + def _check_driver_build_up_to_date(self, configuration): if configuration in ('Debug', 'Release'): try: @@ -400,10 +437,6 @@ class ChromiumPort(Port): return cygpath(path) return path - def _path_to_image_diff(self): - binary_name = 'ImageDiff' - return self._build_path(self.get_option('configuration'), binary_name) - class ChromiumDriver(WebKitDriver): KILL_TIMEOUT_DEFAULT = 3.0 @@ -663,7 +696,7 @@ class ChromiumDriver(WebKitDriver): self._proc.stderr.close() time_out_ms = self._port.get_option('time_out_ms') if time_out_ms and not self._no_timeout: - timeout_ratio = float(time_out_ms) / self._port.default_test_timeout_ms() + timeout_ratio = float(time_out_ms) / self._port.default_timeout_ms() kill_timeout_seconds = self.KILL_TIMEOUT_DEFAULT * timeout_ratio if timeout_ratio > 1.0 else self.KILL_TIMEOUT_DEFAULT else: kill_timeout_seconds = self.KILL_TIMEOUT_DEFAULT diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py index 75b9932db..2a1877407 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py @@ -57,7 +57,9 @@ DEVICE_DRT_STAMP_PATH = DEVICE_DRT_DIR + 'DumpRenderTree.stamp' DRT_APP_PACKAGE = 'org.chromium.native_test' DRT_ACTIVITY_FULL_NAME = DRT_APP_PACKAGE + '/.ChromeNativeTestActivity' -DRT_APP_FILE_DIR = '/data/user/0/' + DRT_APP_PACKAGE + '/files/' +DRT_APP_DIR = '/data/user/0/' + DRT_APP_PACKAGE + '/' +DRT_APP_FILES_DIR = DRT_APP_DIR + 'files/' +DRT_APP_CACHE_DIR = DRT_APP_DIR + 'cache/' # This only works for single core devices so far. # FIXME: Find a solution for multi-core devices. @@ -178,7 +180,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort): self._adb_command += shlex.split(adb_args) self._drt_retry_after_killed = 0 - def default_test_timeout_ms(self): + def default_timeout_ms(self): # Android platform has less computing power than desktop platforms. # Using 10 seconds allows us to pass most slow tests which are not # marked as slow tests on desktop platforms. @@ -214,7 +216,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort): return True # FIXME: Remove this function when chromium-android is fully upstream. - def expectations_files(self): + def expectations_files(self): android_expectations_file = self.path_from_webkit_base('LayoutTests', 'platform', 'chromium', 'test_expectations_android.txt') return super(ChromiumAndroidPort, self).expectations_files() + [android_expectations_file] @@ -227,7 +229,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort): expectations = chromium.ChromiumPort.test_expectations(self) return expectations.replace('LINUX ', 'LINUX ANDROID ') - def start_http_server(self, additional_dirs=None): + def start_http_server(self, additional_dirs=None, number_of_servers=0): # The http server runs during the whole testing period, so ignore this call. pass @@ -235,7 +237,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort): # Same as start_http_server(). pass - def start_helper(self): + def setup_test_run(self): self._run_adb_command(['root']) self._setup_performance() # Required by webkit_support::GetWebKitRootDirFilePath(). @@ -249,13 +251,17 @@ class ChromiumAndroidPort(chromium.ChromiumPort): self._push_fonts() self._synchronize_datetime() + # Delete the disk cache if any to ensure a clean test run. + # This is like what's done in ChromiumPort.setup_test_run but on the device. + self._run_adb_command(['shell', 'rm', '-r', DRT_APP_CACHE_DIR]) + # Start the HTTP server so that the device can access the test cases. chromium.ChromiumPort.start_http_server(self, additional_dirs={TEST_PATH_PREFIX: self.layout_tests_dir()}) _log.debug('Starting forwarder') - cmd = self._run_adb_command(['shell', '%s %s' % (DEVICE_FORWARDER_PATH, FORWARD_PORTS)]) + self._run_adb_command(['shell', '%s %s' % (DEVICE_FORWARDER_PATH, FORWARD_PORTS)]) - def stop_helper(self): + def clean_up_test_run(self): # Leave the forwarder and tests httpd server there because they are # useful for debugging and do no harm to subsequent tests. self._teardown_performance() @@ -284,6 +290,9 @@ class ChromiumAndroidPort(chromium.ChromiumPort): return self._build_path(configuration, 'DumpRenderTree_apk/DumpRenderTree-debug.apk') def _path_to_helper(self): + return None + + def _path_to_forwarder(self): return self._build_path(self.get_option('configuration'), 'forwarder') def _path_to_image_diff(self): @@ -320,7 +329,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort): def _push_executable(self): drt_host_path = self._path_to_driver() - forwarder_host_path = self._path_to_helper() + forwarder_host_path = self._path_to_forwarder() host_stamp = int(float(max(os.stat(drt_host_path).st_mtime, os.stat(forwarder_host_path).st_mtime))) device_stamp = int(float(self._run_adb_command([ @@ -452,9 +461,9 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver): def __init__(self, port, worker_number, pixel_tests, no_timeout=False): chromium.ChromiumDriver.__init__(self, port, worker_number, pixel_tests, no_timeout) - self._in_fifo_path = DRT_APP_FILE_DIR + 'DumpRenderTree.in' - self._out_fifo_path = DRT_APP_FILE_DIR + 'DumpRenderTree.out' - self._err_file_path = DRT_APP_FILE_DIR + 'DumpRenderTree.err' + self._in_fifo_path = DRT_APP_FILES_DIR + 'DumpRenderTree.in' + self._out_fifo_path = DRT_APP_FILES_DIR + 'DumpRenderTree.out' + self._err_file_path = DRT_APP_FILES_DIR + 'DumpRenderTree.err' self._restart_after_killed = False self._read_fifo_proc = None @@ -467,7 +476,7 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver): cmd = [] for param in original_cmd: if param.startswith('--pixel-tests='): - self._device_image_path = DRT_APP_FILE_DIR + self._port.host.filesystem.basename(self._image_path) + self._device_image_path = DRT_APP_FILES_DIR + self._port.host.filesystem.basename(self._image_path) param = '--pixel-tests=' + self._device_image_path cmd.append(param) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py index e0d308267..8544b020c 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py @@ -26,6 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import optparse import StringIO import unittest @@ -35,11 +36,11 @@ from webkitpy.common.system.systemhost_mock import MockSystemHost from webkitpy.thirdparty.mock import Mock from webkitpy.layout_tests.port import chromium_android -from webkitpy.layout_tests.port import port_testcase +from webkitpy.layout_tests.port import chromium_port_testcase from webkitpy.layout_tests.port import Port -class ChromiumAndroidPortTest(port_testcase.PortTestCase): +class ChromiumAndroidPortTest(chromium_port_testcase.ChromiumPortTestCase): port_name = 'chromium-android' port_maker = chromium_android.ChromiumAndroidPort mock_logcat = '' @@ -49,6 +50,15 @@ class ChromiumAndroidPortTest(port_testcase.PortTestCase): self.assertTrue(port.get_option('enable_hardware_gpu')) self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-android')) + def test_default_timeout_ms(self): + self.assertEquals(self.make_port(options=optparse.Values({'configuration': 'Release'})).default_timeout_ms(), 10000) + self.assertEquals(self.make_port(options=optparse.Values({'configuration': 'Debug'})).default_timeout_ms(), 10000) + + def test_expectations_files(self): + # FIXME: override this test temporarily while we're still upstreaming the android port and + # using a custom expectations file. + pass + @staticmethod def mock_run_command_fn(args): if args[1] == 'shell': @@ -123,9 +133,9 @@ class ChromiumAndroidDriverTest(unittest.TestCase): def test_cmd_line(self): cmd_line = self.driver.cmd_line(True, ['--a']) self.assertTrue('--a' in cmd_line) - self.assertTrue('--in-fifo=' + chromium_android.DRT_APP_FILE_DIR + 'DumpRenderTree.in' in cmd_line) - self.assertTrue('--out-fifo=' + chromium_android.DRT_APP_FILE_DIR + 'DumpRenderTree.out' in cmd_line) - self.assertTrue('--err-file=' + chromium_android.DRT_APP_FILE_DIR + 'DumpRenderTree.err' in cmd_line) + self.assertTrue('--in-fifo=' + chromium_android.DRT_APP_FILES_DIR + 'DumpRenderTree.in' in cmd_line) + self.assertTrue('--out-fifo=' + chromium_android.DRT_APP_FILES_DIR + 'DumpRenderTree.out' in cmd_line) + self.assertTrue('--err-file=' + chromium_android.DRT_APP_FILES_DIR + 'DumpRenderTree.err' in cmd_line) def test_read_prompt(self): self.driver._proc = Mock() # FIXME: This should use a tighter mock. diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index f26bea431..e54078d2b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -110,6 +110,9 @@ class ChromiumLinuxPort(chromium.ChromiumPort): port_names = self.FALLBACK_PATHS[self._architecture] return map(self._webkit_baseline_path, port_names) + def _modules_to_search_for_symbols(self): + return [self._build_path(self.get_option('configuration'), 'libffmpegsumo.so')] + def check_build(self, needs_http): result = chromium.ChromiumPort.check_build(self, needs_http) if not result: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py index e0ef728bd..9094d7458 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py @@ -33,10 +33,10 @@ from webkitpy.common.system.systemhost_mock import MockSystemHost from webkitpy.tool.mocktool import MockOptions from webkitpy.layout_tests.port import chromium_linux -from webkitpy.layout_tests.port import port_testcase +from webkitpy.layout_tests.port import chromium_port_testcase -class ChromiumLinuxPortTest(port_testcase.PortTestCase): +class ChromiumLinuxPortTest(chromium_port_testcase.ChromiumPortTestCase): port_name = 'chromium-linux' port_maker = chromium_linux.ChromiumLinuxPort diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index f748cd18a..df1ac7b58 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -39,17 +39,10 @@ _log = logging.getLogger(__name__) class ChromiumMacPort(chromium.ChromiumPort): - SUPPORTED_OS_VERSIONS = ('leopard', 'snowleopard', 'lion', 'future') + SUPPORTED_OS_VERSIONS = ('snowleopard', 'lion', 'future') port_name = 'chromium-mac' FALLBACK_PATHS = { - 'leopard': [ - 'chromium-mac-leopard', - 'chromium-mac-snowleopard', - 'chromium-mac', - 'chromium', - 'mac', - ], 'snowleopard': [ 'chromium-mac-snowleopard', 'chromium-mac', @@ -78,16 +71,16 @@ class ChromiumMacPort(chromium.ChromiumPort): def __init__(self, host, port_name, **kwargs): chromium.ChromiumPort.__init__(self, host, port_name, **kwargs) - - # We're a little generic here because this code is reused by the - # 'google-chrome' port as well as the 'mock-' and 'dryrun-' ports. - self._version = port_name[port_name.index('-mac-') + len('-mac-'):] + self._version = port_name[port_name.index('chromium-mac-') + len('chromium-mac-'):] assert self._version in self.SUPPORTED_OS_VERSIONS def baseline_search_path(self): fallback_paths = self.FALLBACK_PATHS return map(self._webkit_baseline_path, fallback_paths[self._version]) + def _modules_to_search_for_symbols(self): + return [self._build_path(self.get_option('configuration'), 'ffmpegsumo.so')] + def check_build(self, needs_http): result = chromium.ChromiumPort.check_build(self, needs_http) if not result: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py index 77cee2b32..87904a804 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py @@ -29,13 +29,13 @@ import unittest from webkitpy.layout_tests.port import chromium_mac -from webkitpy.layout_tests.port import port_testcase +from webkitpy.layout_tests.port import chromium_port_testcase from webkitpy.tool.mocktool import MockOptions -class ChromiumMacPortTest(port_testcase.PortTestCase): +class ChromiumMacPortTest(chromium_port_testcase.ChromiumPortTestCase): os_name = 'mac' - os_version = 'leopard' + os_version = 'snowleopard' port_name = 'chromium-mac' port_maker = chromium_mac.ChromiumMacPort @@ -44,12 +44,7 @@ class ChromiumMacPortTest(port_testcase.PortTestCase): self.assertEquals(expected, port.name()) def test_versions(self): - self.assertTrue(self.make_port().name() in ('chromium-mac-leopard', 'chromium-mac-snowleopard', 'chromium-mac-lion', 'chromium-mac-future')) - - self.assert_name(None, 'leopard', 'chromium-mac-leopard') - self.assert_name('chromium-mac', 'leopard', 'chromium-mac-leopard') - self.assert_name('chromium-mac-leopard', 'leopard', 'chromium-mac-leopard') - self.assert_name('chromium-mac-leopard', 'snowleopard', 'chromium-mac-leopard') + self.assertTrue(self.make_port().name() in ('chromium-mac-snowleopard', 'chromium-mac-lion', 'chromium-mac-future')) self.assert_name(None, 'snowleopard', 'chromium-mac-snowleopard') self.assert_name('chromium-mac', 'snowleopard', 'chromium-mac-snowleopard') @@ -67,9 +62,6 @@ class ChromiumMacPortTest(port_testcase.PortTestCase): self.assertRaises(AssertionError, self.assert_name, None, 'tiger', 'should-raise-assertion-so-this-value-does-not-matter') def test_baseline_path(self): - port = self.make_port(port_name='chromium-mac-leopard') - self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-mac-leopard')) - port = self.make_port(port_name='chromium-mac-snowleopard') self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-mac-snowleopard')) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py new file mode 100644 index 000000000..697c27242 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py @@ -0,0 +1,205 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system import logtesting +from webkitpy.common.system.executive_mock import MockExecutive2 +from webkitpy.common.system.systemhost_mock import MockSystemHost +from webkitpy.tool.mocktool import MockOptions + +import chromium_android +import chromium_linux +import chromium_mac +import chromium_win + +from webkitpy.layout_tests.models.test_configuration import TestConfiguration +from webkitpy.layout_tests.port import port_testcase + + +class ChromiumPortTestCase(port_testcase.PortTestCase): + + def test_default_timeout_ms(self): + self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000) + self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 12000) + + def test_default_pixel_tests(self): + self.assertEquals(self.make_port().default_pixel_tests(), True) + + def test_missing_symbol_to_skipped_tests(self): + # Test that we get the chromium skips and not the webkit default skips + port = self.make_port() + skip_dict = port._missing_symbol_to_skipped_tests() + self.assertTrue('ff_mp3_decoder' in skip_dict) + self.assertFalse('WebGLShader' in skip_dict) + + def test_all_test_configurations(self): + """Validate the complete set of configurations this port knows about.""" + port = self.make_port() + self.assertEquals(set(port.all_test_configurations()), set([ + TestConfiguration('icecreamsandwich', 'x86', 'debug'), + TestConfiguration('icecreamsandwich', 'x86', 'release'), + TestConfiguration('snowleopard', 'x86', 'debug'), + TestConfiguration('snowleopard', 'x86', 'release'), + TestConfiguration('lion', 'x86', 'debug'), + TestConfiguration('lion', 'x86', 'release'), + TestConfiguration('xp', 'x86', 'debug'), + TestConfiguration('xp', 'x86', 'release'), + TestConfiguration('win7', 'x86', 'debug'), + TestConfiguration('win7', 'x86', 'release'), + TestConfiguration('lucid', 'x86', 'debug'), + TestConfiguration('lucid', 'x86', 'release'), + TestConfiguration('lucid', 'x86_64', 'debug'), + TestConfiguration('lucid', 'x86_64', 'release'), + ])) + + class TestMacPort(chromium_mac.ChromiumMacPort): + def __init__(self, options=None): + options = options or MockOptions() + chromium_mac.ChromiumMacPort.__init__(self, MockSystemHost(os_name='mac', os_version='leopard'), 'chromium-mac-leopard', options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + + class TestAndroidPort(chromium_android.ChromiumAndroidPort): + def __init__(self, options=None): + options = options or MockOptions() + chromium_win.ChromiumAndroidPort.__init__(self, MockSystemHost(os_name='android', os_version='icecreamsandwich'), 'chromium-android', options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + + class TestLinuxPort(chromium_linux.ChromiumLinuxPort): + def __init__(self, options=None): + options = options or MockOptions() + chromium_linux.ChromiumLinuxPort.__init__(self, MockSystemHost(os_name='linux', os_version='lucid'), 'chromium-linux-x86', options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + + class TestWinPort(chromium_win.ChromiumWinPort): + def __init__(self, options=None): + options = options or MockOptions() + chromium_win.ChromiumWinPort.__init__(self, MockSystemHost(os_name='win', os_version='xp'), 'chromium-win-xp', options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + + def test_default_configuration(self): + mock_options = MockOptions() + port = ChromiumPortTestCase.TestLinuxPort(options=mock_options) + self.assertEquals(mock_options.configuration, 'default') + self.assertTrue(port.default_configuration_called) + + mock_options = MockOptions(configuration=None) + port = ChromiumPortTestCase.TestLinuxPort(mock_options) + self.assertEquals(mock_options.configuration, 'default') + self.assertTrue(port.default_configuration_called) + + def test_diff_image(self): + class TestPort(ChromiumPortTestCase.TestLinuxPort): + def _path_to_image_diff(self): + return "/path/to/image_diff" + + port = ChromiumPortTestCase.TestLinuxPort() + mock_image_diff = "MOCK Image Diff" + + def mock_run_command(args): + port._filesystem.write_binary_file(args[4], mock_image_diff) + return 1 + + # Images are different. + port._executive = MockExecutive2(run_command_fn=mock_run_command) + self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0]) + + # Images are the same. + port._executive = MockExecutive2(exit_code=0) + self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL")[0]) + + # There was some error running image_diff. + port._executive = MockExecutive2(exit_code=2) + exception_raised = False + try: + port.diff_image("EXPECTED", "ACTUAL") + except ValueError, e: + exception_raised = True + self.assertFalse(exception_raised) + + def test_expectations_files(self): + port = self.make_port() + port.port_name = 'chromium' + + expectations_path = port.path_to_test_expectations_file() + chromium_overrides_path = port.path_from_chromium_base( + 'webkit', 'tools', 'layout_tests', 'test_expectations.txt') + skia_overrides_path = port.path_from_chromium_base( + 'skia', 'skia_test_expectations.txt') + + port._filesystem.write_text_file(skia_overrides_path, 'dummay text') + + port._options.builder_name = 'DUMMY_BUILDER_NAME' + self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path, chromium_overrides_path]) + + port._options.builder_name = 'builder (deps)' + self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path, chromium_overrides_path]) + + # A builder which does NOT observe the Chromium test_expectations, + # but still observes the Skia test_expectations... + port._options.builder_name = 'builder' + self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path]) + + def test_expectations_ordering(self): + # since we don't implement self.port_name in ChromiumPort. + pass + + +class ChromiumPortLoggingTest(logtesting.LoggingTestCase): + def test_check_sys_deps(self): + port = ChromiumPortTestCase.TestLinuxPort() + + # Success + port._executive = MockExecutive2(exit_code=0) + self.assertTrue(port.check_sys_deps(needs_http=False)) + + # Failure + port._executive = MockExecutive2(exit_code=1, + output='testing output failure') + self.assertFalse(port.check_sys_deps(needs_http=False)) + self.assertLog([ + 'ERROR: System dependencies check failed.\n', + 'ERROR: To override, invoke with --nocheck-sys-deps\n', + 'ERROR: \n', + 'ERROR: testing output failure\n']) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index 492deedf2..0c49112ba 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -27,26 +27,20 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import StringIO -import sys import time import unittest from webkitpy.common.system import logtesting -from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 -from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.executive_mock import MockExecutive2 from webkitpy.common.system.systemhost_mock import MockSystemHost from webkitpy.layout_tests.port.config_mock import MockConfig from webkitpy.thirdparty.mock import Mock from webkitpy.tool.mocktool import MockOptions import chromium -import chromium_android -import chromium_linux import chromium_mac -import chromium_win -from webkitpy.layout_tests.models.test_configuration import TestConfiguration -from webkitpy.layout_tests.port import port_testcase +from webkitpy.layout_tests.port import chromium_port_testcase from webkitpy.layout_tests.port.driver import DriverInput @@ -176,149 +170,20 @@ class ChromiumDriverTest(unittest.TestCase): self.driver._start(True, []) self.assertFalse(self.port._filesystem.isdir(last_tmpdir)) -class ChromiumPortTest(port_testcase.PortTestCase): - port_name = 'chromium-mac' - port_maker = chromium.ChromiumPort - - def test_all_test_configurations(self): - """Validate the complete set of configurations this port knows about.""" - port = self.make_port() - self.assertEquals(set(port.all_test_configurations()), set([ - TestConfiguration('icecreamsandwich', 'x86', 'debug'), - TestConfiguration('icecreamsandwich', 'x86', 'release'), - TestConfiguration('leopard', 'x86', 'debug'), - TestConfiguration('leopard', 'x86', 'release'), - TestConfiguration('snowleopard', 'x86', 'debug'), - TestConfiguration('snowleopard', 'x86', 'release'), - TestConfiguration('lion', 'x86', 'debug'), - TestConfiguration('lion', 'x86', 'release'), - TestConfiguration('xp', 'x86', 'debug'), - TestConfiguration('xp', 'x86', 'release'), - TestConfiguration('win7', 'x86', 'debug'), - TestConfiguration('win7', 'x86', 'release'), - TestConfiguration('lucid', 'x86', 'debug'), - TestConfiguration('lucid', 'x86', 'release'), - TestConfiguration('lucid', 'x86_64', 'debug'), - TestConfiguration('lucid', 'x86_64', 'release'), - ])) - - def test_driver_cmd_line(self): - # Override this test since ChromiumPort doesn't implement driver_cmd_line(). - pass - - def test_check_build(self): - # Override this test since ChromiumPort doesn't implement _path_to_driver(). - pass - - def test_check_wdiff(self): - # Override this test since ChromiumPort doesn't implement _path_to_driver(). - pass - - class TestMacPort(chromium_mac.ChromiumMacPort): - def __init__(self, options=None): - options = options or MockOptions() - chromium_mac.ChromiumMacPort.__init__(self, MockSystemHost(os_name='mac', os_version='leopard'), 'chromium-mac-leopard', options=options) - - def default_configuration(self): - self.default_configuration_called = True - return 'default' - - class TestAndroidPort(chromium_android.ChromiumAndroidPort): - def __init__(self, options=None): - options = options or MockOptions() - chromium_win.ChromiumAndroidPort.__init__(self, MockSystemHost(os_name='android', os_version='icecreamsandwich'), 'chromium-android', options=options) - - def default_configuration(self): - self.default_configuration_called = True - return 'default' - - class TestLinuxPort(chromium_linux.ChromiumLinuxPort): - def __init__(self, options=None): - options = options or MockOptions() - chromium_linux.ChromiumLinuxPort.__init__(self, MockSystemHost(os_name='linux', os_version='lucid'), 'chromium-linux-x86', options=options) - - def default_configuration(self): - self.default_configuration_called = True - return 'default' - - class TestWinPort(chromium_win.ChromiumWinPort): - def __init__(self, options=None): - options = options or MockOptions() - chromium_win.ChromiumWinPort.__init__(self, MockSystemHost(os_name='win', os_version='xp'), 'chromium-win-xp', options=options) - - def default_configuration(self): - self.default_configuration_called = True - return 'default' - - def test_default_configuration(self): - mock_options = MockOptions() - port = ChromiumPortTest.TestLinuxPort(options=mock_options) - self.assertEquals(mock_options.configuration, 'default') - self.assertTrue(port.default_configuration_called) - - mock_options = MockOptions(configuration=None) - port = ChromiumPortTest.TestLinuxPort(mock_options) - self.assertEquals(mock_options.configuration, 'default') - self.assertTrue(port.default_configuration_called) - - def test_diff_image(self): - class TestPort(ChromiumPortTest.TestLinuxPort): - def _path_to_image_diff(self): - return "/path/to/image_diff" - - port = ChromiumPortTest.TestLinuxPort() - mock_image_diff = "MOCK Image Diff" - - def mock_run_command(args): - port._filesystem.write_binary_file(args[4], mock_image_diff) - return 1 - - # Images are different. - port._executive = MockExecutive2(run_command_fn=mock_run_command) - self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0]) - - # Images are the same. - port._executive = MockExecutive2(exit_code=0) - self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL")[0]) - - # There was some error running image_diff. - port._executive = MockExecutive2(exit_code=2) - exception_raised = False - try: - port.diff_image("EXPECTED", "ACTUAL") - except ValueError, e: - exception_raised = True - self.assertFalse(exception_raised) - - def test_expectations_files(self): - port = self.make_port() - port.port_name = 'chromium' - - expectations_path = port.path_to_test_expectations_file() - chromium_overrides_path = port.path_from_chromium_base( - 'webkit', 'tools', 'layout_tests', 'test_expectations.txt') - skia_overrides_path = port.path_from_chromium_base( - 'skia', 'skia_test_expectations.txt') + def test_expectations_dict(self): + self.port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/chromium/TestExpectations', 'upstream') + self.port._filesystem.write_text_file('/mock-checkout/Source/WebKit/chromium/webkit/tools/layout_tests/test_expectations.txt', 'downstream') + self.assertEquals('\n'.join(self.port.expectations_dict().values()), 'upstream\ndownstream') - port._options.builder_name = 'DUMMY_BUILDER_NAME' - self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path, chromium_overrides_path]) - - port._options.builder_name = 'builder (deps)' - self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path, chromium_overrides_path]) - - # A builder which does NOT observe the Chromium test_expectations, - # but still observes the Skia test_expectations... - port._options.builder_name = 'builder' - self.assertEquals(port.expectations_files(), [expectations_path, skia_overrides_path]) - - def test_expectations_ordering(self): - # since we don't implement self.port_name in ChromiumPort. - pass + self.port._filesystem.write_text_file(self.port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), 'skia') + self.assertEquals('\n'.join(self.port.expectations_dict().values()), 'upstream\nskia\ndownstream') class ChromiumPortLoggingTest(logtesting.LoggingTestCase): + + # FIXME: put this someplace more useful def test_check_sys_deps(self): - port = ChromiumPortTest.TestLinuxPort() + port = chromium_port_testcase.ChromiumPortTestCase.TestLinuxPort() # Success port._executive = MockExecutive2(exit_code=0) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py index ff58842b6..51611241c 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -75,10 +75,7 @@ class ChromiumWinPort(chromium.ChromiumPort): def __init__(self, host, port_name, **kwargs): chromium.ChromiumPort.__init__(self, host, port_name, **kwargs) - - # We're a little generic here because this code is reused by the - # 'google-chrome' port as well as the 'mock-' and 'dryrun-' ports. - self._version = port_name[port_name.index('-win-') + len('-win-'):] + self._version = port_name[port_name.index('chromium-win-') + len('chromium-win-'):] assert self._version in self.SUPPORTED_VERSIONS, "%s is not in %s" % (self._version, self.SUPPORTED_VERSIONS) def setup_environ_for_server(self, server_name=None): @@ -106,6 +103,11 @@ class ChromiumWinPort(chromium.ChromiumPort): port_names = self.FALLBACK_PATHS[self.version()] return map(self._webkit_baseline_path, port_names) + def _modules_to_search_for_symbols(self): + # FIXME: we should return the path to the ffmpeg equivalents to detect if we have the mp3 and aac codecs installed. + # See https://bugs.webkit.org/show_bug.cgi?id=89706. + return [] + def check_build(self, needs_http): result = chromium.ChromiumPort.check_build(self, needs_http) if not result: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py index ac2358194..607719241 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py @@ -32,12 +32,12 @@ import unittest from webkitpy.common.system import outputcapture from webkitpy.common.system.executive_mock import MockExecutive from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.layout_tests.port import chromium_port_testcase from webkitpy.layout_tests.port import chromium_win -from webkitpy.layout_tests.port import port_testcase from webkitpy.tool.mocktool import MockOptions -class ChromiumWinTest(port_testcase.PortTestCase): +class ChromiumWinTest(chromium_port_testcase.ChromiumPortTestCase): port_name = 'chromium-win' port_maker = chromium_win.ChromiumWinPort os_name = 'win' diff --git a/Tools/Scripts/webkitpy/layout_tests/port/efl.py b/Tools/Scripts/webkitpy/layout_tests/port/efl.py index 8ea8a1255..5964dfe52 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/efl.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/efl.py @@ -52,6 +52,12 @@ class EflPort(WebKitPort, PulseAudioSanitizer): def setup_test_run(self): self._unload_pulseaudio_module() + def setup_environ_for_server(self, server_name=None): + env = super(EflPort, self).setup_environ_for_server(server_name) + env['TEST_RUNNER_INJECTED_BUNDLE_FILENAME'] = self._build_path('lib', 'libTestRunnerInjectedBundle.so') + env['TEST_RUNNER_PLUGIN_PATH'] = self._build_path('lib') + return env + def clean_up_test_run(self): self._restore_pulseaudio_module() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory.py b/Tools/Scripts/webkitpy/layout_tests/port/factory.py index 881a80bec..9b5cf27b0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory.py @@ -60,8 +60,9 @@ def port_options(**help_strings): def _builder_options(builder_name): configuration = "Debug" if re.search(r"[d|D](ebu|b)g", builder_name) else "Release" + is_webkit2 = builder_name.find("WK2") != -1 builder_name = builder_name - return optparse.Values({'builder_name': builder_name, 'configuration': configuration}) + return optparse.Values({'builder_name': builder_name, 'configuration': configuration, 'webkit_test_runner': is_webkit2}) class PortFactory(object): @@ -71,10 +72,6 @@ class PortFactory(object): 'chromium_mac.ChromiumMacPort', 'chromium_win.ChromiumWinPort', 'efl.EflPort', - 'google_chrome.GoogleChromeLinux32Port', - 'google_chrome.GoogleChromeLinux64Port', - 'google_chrome.GoogleChromeMacPort', - 'google_chrome.GoogleChromeWinPort', 'gtk.GtkPort', 'mac.MacPort', 'mock_drt.MockDRTPort', @@ -115,7 +112,7 @@ class PortFactory(object): if port_name.startswith(cls.port_name): port_name = cls.determine_full_port_name(self._host, options, port_name) return cls(self._host, port_name, options=options, **kwargs) - raise NotImplementedError('unsupported port: %s' % port_name) + raise NotImplementedError('unsupported platform: "%s"' % port_name) def all_port_names(self): """Return a list of all valid, fully-specified, "real" port names. diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index 700399e28..776433f80 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -35,7 +35,6 @@ from webkitpy.layout_tests.port import chromium_linux from webkitpy.layout_tests.port import chromium_mac from webkitpy.layout_tests.port import chromium_win from webkitpy.layout_tests.port import factory -from webkitpy.layout_tests.port import google_chrome from webkitpy.layout_tests.port import gtk from webkitpy.layout_tests.port import mac from webkitpy.layout_tests.port import qt @@ -59,8 +58,8 @@ class FactoryTest(unittest.TestCase): def test_mac(self): self.assert_port(port_name='mac-leopard', cls=mac.MacPort) self.assert_port(port_name='mac-leopard-wk2', cls=mac.MacPort) - self.assert_port(port_name='mac', os_name='mac', os_version='leopard', cls=mac.MacPort) - self.assert_port(port_name=None, os_name='mac', os_version='leopard', cls=mac.MacPort) + self.assert_port(port_name='mac', os_name='mac', os_version='lion', cls=mac.MacPort) + self.assert_port(port_name=None, os_name='mac', os_version='lion', cls=mac.MacPort) def test_win(self): self.assert_port(port_name='win-xp', cls=win.WinPort) @@ -69,26 +68,6 @@ class FactoryTest(unittest.TestCase): self.assert_port(port_name=None, os_name='win', os_version='xp', cls=win.WinPort) self.assert_port(port_name=None, os_name='win', os_version='xp', options=self.webkit_options, cls=win.WinPort) - def test_google_chrome(self): - self.assert_port(port_name='google-chrome-linux32', - cls=google_chrome.GoogleChromeLinux32Port) - self.assert_port(port_name='google-chrome-linux64', os_name='linux', os_version='lucid', - cls=google_chrome.GoogleChromeLinux64Port) - self.assert_port(port_name='google-chrome-linux64', - cls=google_chrome.GoogleChromeLinux64Port) - self.assert_port(port_name='google-chrome-win-xp', - cls=google_chrome.GoogleChromeWinPort) - self.assert_port(port_name='google-chrome-win', os_name='win', os_version='xp', - cls=google_chrome.GoogleChromeWinPort) - self.assert_port(port_name='google-chrome-win-xp', os_name='win', os_version='xp', - cls=google_chrome.GoogleChromeWinPort) - self.assert_port(port_name='google-chrome-mac', os_name='mac', os_version='leopard', - cls=google_chrome.GoogleChromeMacPort) - self.assert_port(port_name='google-chrome-mac-leopard', os_name='mac', os_version='leopard', - cls=google_chrome.GoogleChromeMacPort) - self.assert_port(port_name='google-chrome-mac-leopard', - cls=google_chrome.GoogleChromeMacPort) - def test_gtk(self): self.assert_port(port_name='gtk', cls=gtk.GtkPort) @@ -96,10 +75,9 @@ class FactoryTest(unittest.TestCase): self.assert_port(port_name='qt', cls=qt.QtPort) def test_chromium_mac(self): - self.assert_port(port_name='chromium-mac-leopard', cls=chromium_mac.ChromiumMacPort) - self.assert_port(port_name='chromium-mac', os_name='mac', os_version='leopard', + self.assert_port(port_name='chromium-mac', os_name='mac', os_version='snowleopard', cls=chromium_mac.ChromiumMacPort) - self.assert_port(port_name='chromium', os_name='mac', os_version='leopard', + self.assert_port(port_name='chromium', os_name='mac', os_version='lion', cls=chromium_mac.ChromiumMacPort) def test_chromium_linux(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py deleted file mode 100644 index 8d2df0ee9..000000000 --- a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import chromium_linux -import chromium_mac -import chromium_win - - -def _expectations_files(port, super): - return super.expectations_files(port) + [port.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations_chrome.txt')] - - -class GoogleChromeLinux32Port(chromium_linux.ChromiumLinuxPort): - port_name = 'google-chrome-linux32' - - # FIXME: Make google-chrome-XXX work as a port name. - @classmethod - def determine_full_port_name(cls, host, options, port_name): - return 'chromium-linux-x86' - - def baseline_search_path(self): - paths = chromium_linux.ChromiumLinuxPort.baseline_search_path(self) - paths.insert(0, self._webkit_baseline_path('google-chrome-linux32')) - return paths - - def expectations_files(self): - return _expectations_files(self, chromium_linux.ChromiumLinuxPort) - - def architecture(self): - return 'x86' - - -class GoogleChromeLinux64Port(chromium_linux.ChromiumLinuxPort): - port_name = 'google-chrome-linux64' - - # FIXME: Make google-chrome-XXX work as a port name. - @classmethod - def determine_full_port_name(cls, host, options, port_name): - return 'chromium-linux-x86_64' - - def baseline_search_path(self): - paths = chromium_linux.ChromiumLinuxPort.baseline_search_path(self) - paths.insert(0, self._webkit_baseline_path('google-chrome-linux64')) - return paths - - def expectations_files(self): - return _expectations_files(self, chromium_linux.ChromiumLinuxPort) - - def architecture(self): - return 'x86_64' - - -class GoogleChromeMacPort(chromium_mac.ChromiumMacPort): - port_name = 'google-chrome-mac' - - # FIXME: Make google-chrome-XXX work as a port name. - @classmethod - def determine_full_port_name(cls, host, options, port_name): - return 'chromium-mac-snowleopard' - - def baseline_search_path(self): - paths = chromium_mac.ChromiumMacPort.baseline_search_path(self) - paths.insert(0, self._webkit_baseline_path('google-chrome-mac')) - return paths - - def expectations_files(self): - return _expectations_files(self, chromium_mac.ChromiumMacPort) - - -class GoogleChromeWinPort(chromium_win.ChromiumWinPort): - port_name = 'google-chrome-win' - - # FIXME: Make google-chrome-XXX work as a port name. - @classmethod - def determine_full_port_name(cls, host, options, port_name): - return 'chromium-win-win7' - - def baseline_search_path(self): - paths = chromium_win.ChromiumWinPort.baseline_search_path(self) - paths.insert(0, self._webkit_baseline_path('google-chrome-win')) - return paths - - def expectations_files(self): - return _expectations_files(self, chromium_win.ChromiumWinPort) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py deleted file mode 100644 index 6d3c1a7dd..000000000 --- a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest - -from webkitpy.common.system.systemhost_mock import MockSystemHost -from webkitpy.layout_tests.port.factory import PortFactory - - -class TestGoogleChromePort(unittest.TestCase): - def _verify_baseline_search_path_startswith(self, port_name, expected_platform_dirs): - port = PortFactory(MockSystemHost()).get(port_name=port_name) - actual_platform_dirs = [port._filesystem.basename(path) for path in port.baseline_search_path()] - self.assertEqual(expected_platform_dirs, actual_platform_dirs[0:len(expected_platform_dirs)]) - - def _verify_expectations_overrides(self, port_name): - host = MockSystemHost() - port = PortFactory(host).get(port_name=port_name, options=None) - self.assertTrue('TestExpectations' in port.expectations_files()[0]) - self.assertTrue('skia_test_expectations.txt' in port.expectations_files()[1]) - self.assertTrue('test_expectations_chrome.txt' in port.expectations_files()[-1]) - - def test_get_google_chrome_port(self): - self._verify_baseline_search_path_startswith('google-chrome-linux32', ['google-chrome-linux32', 'chromium-linux-x86']) - self._verify_baseline_search_path_startswith('google-chrome-linux64', ['google-chrome-linux64', 'chromium-linux']) - self._verify_baseline_search_path_startswith('google-chrome-mac', ['google-chrome-mac', 'chromium-mac-snowleopard']) - self._verify_baseline_search_path_startswith('google-chrome-win', ['google-chrome-win', 'chromium-win']) - - self._verify_expectations_overrides('google-chrome-mac') - self._verify_expectations_overrides('google-chrome-win') - self._verify_expectations_overrides('google-chrome-linux32') - self._verify_expectations_overrides('google-chrome-linux64') - - -if __name__ == '__main__': - unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py index 26e1c84fb..6c59427f3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -64,6 +64,7 @@ class GtkPort(WebKitPort, PulseAudioSanitizer): environment['AUDIO_RESOURCES_PATH'] = self._filesystem.join(self._config.webkit_base_dir(), 'Source', 'WebCore', 'platform', 'audio', 'resources') + self._copy_value_from_environ_if_set(environment, 'WEBKITOUTPUTDIR') if self.get_option('webkit_test_runner'): # FIXME: This is a workaround to ensure that testing with WebKitTestRunner is started with # a non-existing cache. This should be removed when (and if) it will be possible to properly diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py index a3ec6a4b9..372ffb807 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py @@ -65,6 +65,11 @@ class MacPort(ApplePort): # with MallocStackLogging enabled. self.set_option_default("batch_size", 1000) + def default_timeout_ms(self): + if self.get_option('guard_malloc'): + return 350 * 1000 + return super(MacPort, self).default_timeout_ms() + def _build_driver_flags(self): return ['ARCHS=i386'] if self.architecture() == 'x86' else [] diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py index 2ebf255ae..bc7b0df44 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -45,6 +45,10 @@ class MacTest(port_testcase.PortTestCase): port = self.make_port(port_name=port_name, options=MockOptions(webkit_test_runner=use_webkit2)) self.assertEqual(port._skipped_file_search_paths(), expected_paths) + def test_default_timeout_ms(self): + super(MacTest, self).test_default_timeout_ms() + self.assertEquals(self.make_port(options=MockOptions(guard_malloc=True)).default_timeout_ms(), 350000) + def test_skipped_file_search_paths(self): self.assert_skipped_file_search_paths('mac-snowleopard', set(['mac-snowleopard', 'mac'])) self.assert_skipped_file_search_paths('mac-leopard', set(['mac-leopard', 'mac'])) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py index cc3dc7e63..a2106fdd9 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py @@ -39,6 +39,7 @@ MockDRT to crash). import base64 import logging +import optparse import os import sys @@ -51,7 +52,6 @@ if script_dir not in sys.path: from webkitpy.common.system.systemhost import SystemHost from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput, DriverProxy from webkitpy.layout_tests.port.factory import PortFactory -from webkitpy.tool.mocktool import MockOptions _log = logging.getLogger(__name__) @@ -104,7 +104,7 @@ class MockDRTPort(object): def start_helper(self): pass - def start_http_server(self): + def start_http_server(self, number_of_servers): pass def start_websocket_server(self): @@ -130,8 +130,8 @@ def main(argv, host, stdin, stdout, stderr): """Run the tests.""" options, args = parse_options(argv) - if options.chromium: - drt = MockChromiumDRT(options, args, host, stdin, stdout, stderr) + if options.test_shell: + drt = MockTestShell(options, args, host, stdin, stdout, stderr) else: drt = MockDRT(options, args, host, stdin, stdout, stderr) return drt.run() @@ -151,16 +151,15 @@ def parse_options(argv): pixel_tests = False pixel_path = None - chromium = False - if platform.startswith('chromium'): - chromium = True + test_shell = '--test-shell' in argv + if test_shell: for arg in argv: if arg.startswith('--pixel-tests'): pixel_tests = True pixel_path = arg[len('--pixel-tests='):] else: pixel_tests = '--pixel-tests' in argv - options = MockOptions(chromium=chromium, platform=platform, pixel_tests=pixel_tests, pixel_path=pixel_path) + options = optparse.Values({'test_shell': test_shell, 'platform': platform, 'pixel_tests': pixel_tests, 'pixel_path': pixel_path}) return (options, argv) @@ -255,7 +254,7 @@ class MockDRT(object): self._stderr.flush() -class MockChromiumDRT(MockDRT): +class MockTestShell(MockDRT): def input_from_line(self, line): vals = line.strip().split() if len(vals) == 3: @@ -272,7 +271,7 @@ class MockChromiumDRT(MockDRT): original_test_name = test_input.test_name if '--enable-accelerated-2d-canvas' in self._args and 'canvas' in test_input.test_name: test_input.test_name = 'platform/chromium/virtual/gpu/' + test_input.test_name - output = super(MockChromiumDRT, self).output_for_test(test_input, is_reftest) + output = super(MockTestShell, self).output_for_test(test_input, is_reftest) test_input.test_name = original_test_name return output diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py index 570a9e4ad..1654e1c48 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py @@ -150,7 +150,7 @@ class MockDRTTest(unittest.TestCase): # We use the StringIO.buflist here instead of getvalue() because # the StringIO might be a mix of unicode/ascii and 8-bit strings. self.assertEqual(stdout.buflist, drt_output) - self.assertEqual(stderr.getvalue(), '' if options.chromium else '#EOF\n') + self.assertEqual(stderr.getvalue(), '' if options.test_shell else '#EOF\n') def test_main(self): host = MockSystemHost() @@ -201,22 +201,21 @@ class MockDRTTest(unittest.TestCase): self.assertTest('passes/mismatch.html', True, expected_checksum='mock-checksum', expected_text='reference text\n') - -class MockChromiumDRTTest(MockDRTTest): +class MockTestShellTest(MockDRTTest): def extra_args(self, pixel_tests): if pixel_tests: return ['--pixel-tests=/tmp/png_result0.png'] return [] def make_drt(self, options, args, host, stdin, stdout, stderr): - options.chromium = True + options.test_shell = True # We have to set these by hand because --platform test won't trigger # the Chromium code paths. options.pixel_path = '/tmp/png_result0.png' options.pixel_tests = True - return mock_drt.MockChromiumDRT(options, args, host, stdin, stdout, stderr) + return mock_drt.MockTestShell(options, args, host, stdin, stdout, stderr) def input_line(self, port, test_name, checksum=None): url = port.create_driver(0).test_to_uri(test_name) @@ -251,10 +250,10 @@ class MockChromiumDRTTest(MockDRTTest): self.assertEquals(host.filesystem.written_files, {'/tmp/png_result0.png': 'image_checksum\x8a-pngtEXtchecksum\x00image_checksum-checksum'}) - def test_chromium_parse_options(self): - options, args = mock_drt.parse_options(['--platform', 'chromium-mac', + def test_test_shell_parse_options(self): + options, args = mock_drt.parse_options(['--platform', 'chromium-mac', '--test-shell', '--pixel-tests=/tmp/png_result0.png']) - self.assertTrue(options.chromium) + self.assertTrue(options.test_shell) self.assertTrue(options.pixel_tests) self.assertEquals(options.pixel_path, '/tmp/png_result0.png') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py index f37273f78..9f77832aa 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -62,6 +62,13 @@ class PortTestCase(unittest.TestCase): port_name = self.port_maker.determine_full_port_name(host, options, port_name) return self.port_maker(host, port_name, options=options, config=config, **kwargs) + def test_default_timeout_ms(self): + self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 35000) + self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 35000) + + def test_default_pixel_tests(self): + self.assertEquals(self.make_port().default_pixel_tests(), False) + def test_driver_cmd_line(self): port = self.make_port() self.assertTrue(len(port.driver_cmd_line())) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt.py b/Tools/Scripts/webkitpy/layout_tests/port/qt.py index 2488936fe..154cfdab8 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt.py @@ -106,7 +106,16 @@ class QtPort(WebKitPort): version = '4.8' return version - def baseline_search_path(self): + def _search_paths(self): + # Qt port uses same paths for baseline_search_path and _skipped_file_search_paths + # + # qt-5.0-wk1 qt-5.0-wk2 + # \/ + # qt-5.0 qt-4.8 + # \/ + # (qt-linux|qt-mac|qt-win) + # | + # qt search_paths = [] version = self.qt_version() if '5.0' in version: @@ -114,26 +123,27 @@ class QtPort(WebKitPort): search_paths.append('qt-5.0-wk2') else: search_paths.append('qt-5.0-wk1') - search_paths.append(self.name()) if '4.8' in version: search_paths.append('qt-4.8') elif version: search_paths.append('qt-5.0') + search_paths.append(self.port_name + '-' + self.host.platform.os_name) search_paths.append(self.port_name) - return map(self._webkit_baseline_path, search_paths) + return search_paths + + def baseline_search_path(self): + return map(self._webkit_baseline_path, self._search_paths()) def _skipped_file_search_paths(self): - search_paths = set([self.port_name, self.name()]) - version = self.qt_version() - if '4.8' in version: - search_paths.add('qt-4.8') - elif version: - search_paths.add('qt-5.0') - if self.get_option('webkit_test_runner'): - search_paths.update(['qt-5.0-wk2', 'wk2']) - else: - search_paths.add('qt-5.0-wk1') - return search_paths + skipped_path = self._search_paths() + if self.get_option('webkit_test_runner') and '5.0' in self.qt_version(): + skipped_path.append('wk2') + return skipped_path + + def expectations_files(self): + # expectations_files() uses the directories listed in _search_paths reversed. + # e.g. qt -> qt-linux -> qt-4.8 + return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self._search_paths()])) def setup_environ_for_server(self, server_name=None): clean_env = WebKitPort.setup_environ_for_server(self, server_name) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py index 7252b9833..374b10270 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py @@ -28,6 +28,7 @@ import unittest import os +from copy import deepcopy from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 from webkitpy.common.system.outputcapture import OutputCapture @@ -40,8 +41,25 @@ from webkitpy.tool.mocktool import MockOptions class QtPortTest(port_testcase.PortTestCase): port_name = 'qt-mac' port_maker = QtPort + search_paths_cases = [ + {'search_paths':['qt-4.8', 'qt-mac', 'qt'], 'os_name':'mac', 'use_webkit2':False, 'qt_version':'4.8'}, + {'search_paths':['qt-4.8', 'qt-win', 'qt'], 'os_name':'win', 'use_webkit2':False, 'qt_version':'4.8'}, + {'search_paths':['qt-4.8', 'qt-linux', 'qt'], 'os_name':'linux', 'use_webkit2':False, 'qt_version':'4.8'}, - def _assert_search_path(self, search_paths, os_name=None, use_webkit2=False, qt_version='4.8'): + {'search_paths':['qt-4.8', 'qt-mac', 'qt'], 'os_name':'mac', 'use_webkit2':False}, + {'search_paths':['qt-4.8', 'qt-win', 'qt'], 'os_name':'win', 'use_webkit2':False}, + {'search_paths':['qt-4.8', 'qt-linux', 'qt'], 'os_name':'linux', 'use_webkit2':False}, + + {'search_paths':['qt-5.0-wk2', 'qt-5.0', 'qt-mac', 'qt'], 'os_name':'mac', 'use_webkit2':True, 'qt_version':'5.0'}, + {'search_paths':['qt-5.0-wk2', 'qt-5.0', 'qt-win', 'qt'], 'os_name':'win', 'use_webkit2':True, 'qt_version':'5.0'}, + {'search_paths':['qt-5.0-wk2', 'qt-5.0', 'qt-linux', 'qt'], 'os_name':'linux', 'use_webkit2':True, 'qt_version':'5.0'}, + + {'search_paths':['qt-5.0-wk1', 'qt-5.0', 'qt-mac', 'qt'], 'os_name':'mac', 'use_webkit2':False, 'qt_version':'5.0'}, + {'search_paths':['qt-5.0-wk1', 'qt-5.0', 'qt-win', 'qt'], 'os_name':'win', 'use_webkit2':False, 'qt_version':'5.0'}, + {'search_paths':['qt-5.0-wk1', 'qt-5.0', 'qt-linux', 'qt'], 'os_name':'linux', 'use_webkit2':False, 'qt_version':'5.0'}, + ] + + def _assert_search_path(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): # FIXME: Port constructors should not "parse" the port name, but # rather be passed components (directly or via setters). Once # we fix that, this method will need a re-write. @@ -53,6 +71,25 @@ class QtPortTest(port_testcase.PortTestCase): absolute_search_paths = map(port._webkit_baseline_path, search_paths) self.assertEquals(port.baseline_search_path(), absolute_search_paths) + def _assert_skipped_path(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): + host = MockSystemHost(os_name=os_name) + host.executive = MockExecutive2(self._qt_version(qt_version)) + port_name = 'qt-' + os_name + port = self.make_port(host=host, qt_version=qt_version, port_name=port_name, + options=MockOptions(webkit_test_runner=use_webkit2, platform='qt')) + self.assertEquals(port._skipped_file_search_paths(), search_paths) + + def _assert_expectations_files(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): + # FIXME: Port constructors should not "parse" the port name, but + # rather be passed components (directly or via setters). Once + # we fix that, this method will need a re-write. + host = MockSystemHost(os_name=os_name) + host.executive = MockExecutive2(self._qt_version(qt_version)) + port_name = 'qt-' + os_name + port = self.make_port(host=host, qt_version=qt_version, port_name=port_name, + options=MockOptions(webkit_test_runner=use_webkit2, platform='qt')) + self.assertEquals(port.expectations_files(), search_paths) + def _qt_version(self, qt_version): if qt_version in '4.8': return 'QMake version 2.01a\nUsing Qt version 4.8.0 in /usr/local/Trolltech/Qt-4.8.2/lib' @@ -60,21 +97,23 @@ class QtPortTest(port_testcase.PortTestCase): return 'QMake version 2.01a\nUsing Qt version 5.0.0 in /usr/local/Trolltech/Qt-5.0.0/lib' def test_baseline_search_path(self): - self._assert_search_path(['qt-mac', 'qt-4.8', 'qt'], 'mac', qt_version='4.8') - self._assert_search_path(['qt-win', 'qt-4.8', 'qt'], 'win', qt_version='4.8') - self._assert_search_path(['qt-linux', 'qt-4.8', 'qt'], 'linux', qt_version='4.8') - - self._assert_search_path(['qt-mac', 'qt-4.8', 'qt'], 'mac') - self._assert_search_path(['qt-win', 'qt-4.8', 'qt'], 'win') - self._assert_search_path(['qt-linux', 'qt-4.8', 'qt'], 'linux') - - self._assert_search_path(['qt-5.0-wk2', 'qt-mac', 'qt-5.0', 'qt'], 'mac', use_webkit2=True, qt_version='5.0') - self._assert_search_path(['qt-5.0-wk2', 'qt-win', 'qt-5.0', 'qt'], 'win', use_webkit2=True, qt_version='5.0') - self._assert_search_path(['qt-5.0-wk2', 'qt-linux', 'qt-5.0', 'qt'], 'linux', use_webkit2=True, qt_version='5.0') - - self._assert_search_path(['qt-5.0-wk1', 'qt-mac', 'qt-5.0', 'qt'], 'mac', use_webkit2=False, qt_version='5.0') - self._assert_search_path(['qt-5.0-wk1', 'qt-win', 'qt-5.0', 'qt'], 'win', use_webkit2=False, qt_version='5.0') - self._assert_search_path(['qt-5.0-wk1', 'qt-linux', 'qt-5.0', 'qt'], 'linux', use_webkit2=False, qt_version='5.0') + for case in self.search_paths_cases: + self._assert_search_path(**case) + + def test_skipped_file_search_path(self): + caselist = self.search_paths_cases[:] + for case in caselist: + if case['use_webkit2'] and case['qt_version'] == '5.0': + case['search_paths'].append("wk2") + self._assert_skipped_path(**case) + + def test_expectations_files(self): + for case in self.search_paths_cases: + expectations_case = deepcopy(case) + expectations_case['search_paths'] = [] + for path in reversed(case['search_paths']): + expectations_case['search_paths'].append('/mock-checkout/LayoutTests/platform/%s/TestExpectations' % (path)) + self._assert_expectations_files(**expectations_case) def test_show_results_html_file(self): port = self.make_port() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py index 2ca8ea4c3..9cf98de74 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -357,6 +357,9 @@ class TestPort(Port): } self._version = version_map[port_name] + def default_pixel_tests(self): + return True + def _path_to_driver(self): # This routine shouldn't normally be called, but it is called by # the mock_drt Driver. We return something, but make sure it's useless. diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py index 2a1e6e819..fad6f7a5d 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -51,15 +51,6 @@ _log = logging.getLogger(__name__) class WebKitPort(Port): - def __init__(self, host, port_name=None, **kwargs): - Port.__init__(self, host, port_name=port_name, **kwargs) - - # FIXME: Disable pixel tests until they are run by default on build.webkit.org. - self.set_option_default("pixel_tests", False) - # WebKit ports expect a 35s timeout, or 350s timeout when running with -g/--guard-malloc. - # FIXME: --guard-malloc is only supported on Mac, so this logic should be in mac.py. - default_time_out_seconds = 350 if self.get_option('guard_malloc') else 35 - self.set_option_default("time_out_ms", default_time_out_seconds * 1000) def driver_name(self): if self.get_option('webkit_test_runner'): @@ -264,15 +255,20 @@ class WebKitPort(Port): def nm_command(self): return 'nm' - def _webcore_symbols_string(self): - webcore_library_path = self._path_to_webcore_library() - if not webcore_library_path: - return None - try: - return self._executive.run_command([self.nm_command(), webcore_library_path], error_handler=Executive.ignore_error) - except OSError, e: - _log.warn("Failed to run nm: %s. Can't determine WebCore supported features." % e) - return None + def _modules_to_search_for_symbols(self): + path = self._path_to_webcore_library() + if path: + return [path] + return [] + + def _symbols_string(self): + symbols = '' + for path_to_module in self._modules_to_search_for_symbols(): + try: + symbols += self._executive.run_command([self.nm_command(), path_to_module], error_handler=Executive.ignore_error) + except OSError, e: + _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e) + return symbols # Ports which use run-time feature detection should define this method and return # a dictionary mapping from Feature Names to skipped directoires. NRWT will @@ -302,7 +298,7 @@ class WebKitPort(Port): "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"], "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"], "MHTMLArchive": ["mhtml"], - "CSSVariableValue": ["fast/css/variables"], + "CSSVariableValue": ["fast/css/variables", "inspector/styles/variables"], } def _has_test_in_directories(self, directory_lists, test_list): @@ -331,10 +327,10 @@ class WebKitPort(Port): # This is a performance optimization to avoid the calling nm. if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list): # Runtime feature detection not supported, fallback to static dectection: - # Disable any tests for symbols missing from the webcore symbol string. - webcore_symbols_string = self._webcore_symbols_string() - if webcore_symbols_string is not None: - return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in webcore_symbols_string], []) + # Disable any tests for symbols missing from the executable or libraries. + symbols_string = self._symbols_string() + if symbols_string is not None: + return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], []) # Failed to get any runtime or symbol information, don't skip any tests. return [] diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py index f7e9525d6..cfec29b33 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py @@ -53,7 +53,7 @@ class TestWebKitPort(WebKitPort): def all_test_configurations(self): return [self.test_configuration()] - def _webcore_symbols_string(self): + def _symbols_string(self): return self.symbols_string def _tests_for_other_platforms(self): @@ -63,21 +63,6 @@ class TestWebKitPort(WebKitPort): return ["accessibility", ] -class WebKitPortUnitTests(unittest.TestCase): - def test_default_options(self): - # The WebKit ports override new-run-webkit-test default options. - options = MockOptions(pixel_tests=None, time_out_ms=None) - port = WebKitPort(MockSystemHost(), options=options) - self.assertEquals(port._options.pixel_tests, False) - self.assertEquals(port._options.time_out_ms, 35000) - - # Note that we don't override options if specified by the user. - options = MockOptions(pixel_tests=True, time_out_ms=6000) - port = WebKitPort(MockSystemHost(), options=options) - self.assertEquals(port._options.pixel_tests, True) - self.assertEquals(port._options.time_out_ms, 6000) - - class WebKitPortTest(port_testcase.PortTestCase): port_name = 'webkit' port_maker = TestWebKitPort @@ -115,6 +100,7 @@ class WebKitPortTest(port_testcase.PortTestCase): "http/tests/canvas/webgl", # Requires WebGLShader "mhtml", # Requires MHTMLArchive "fast/css/variables", # Requires CSS Variables + "inspector/styles/variables", # Requires CSS Variables ]) result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features(test_list=['mathml/foo.html'])) @@ -127,7 +113,7 @@ class WebKitPortTest(port_testcase.PortTestCase): 000000000124f670 s __ZZN7WebCore13GraphicsLayer13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__ """ # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked): - expected_directories = set(['mathml', 'transforms/3d', 'compositing/webgl', 'fast/canvas/webgl', 'animations/3d', 'mhtml', 'http/tests/canvas/webgl', 'fast/css/variables']) + expected_directories = set(['mathml', 'transforms/3d', 'compositing/webgl', 'fast/canvas/webgl', 'animations/3d', 'mhtml', 'http/tests/canvas/webgl', 'fast/css/variables', 'inspector/styles/variables']) result_directories = set(TestWebKitPort(symbols_string, None)._skipped_tests_for_unsupported_features(test_list=['mathml/foo.html'])) self.assertEqual(result_directories, expected_directories) diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 3d7b20f41..e3a13c20f 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -37,7 +37,8 @@ import signal import sys from webkitpy.common.host import Host -from webkitpy.layout_tests.controllers.manager import Manager, WorkerException +from webkitpy.common.system import stack_utils +from webkitpy.layout_tests.controllers.manager import Manager, WorkerException, TestRunInterruptedException from webkitpy.layout_tests.models import test_expectations from webkitpy.layout_tests.port import port_options from webkitpy.layout_tests.views import printing @@ -46,6 +47,14 @@ from webkitpy.layout_tests.views import printing _log = logging.getLogger(__name__) +# This mirrors what the shell normally does. +INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128 + +# This is a randomly chosen exit code that can be tested against to +# indicate that an unexpected exception occurred. +EXCEPTIONAL_EXIT_STATUS = 254 + + def lint(port, options): host = port.host if options.platform: @@ -119,6 +128,11 @@ def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdo unexpected_result_count = manager.run() _log.debug("Testing completed, Exit status: %d" % unexpected_result_count) + except Exception: + exception_type, exception_value, exception_traceback = sys.exc_info() + if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException): + stack_utils.log_traceback(_log.error, exception_traceback) + raise finally: printer.cleanup() @@ -138,13 +152,10 @@ def _set_up_derived_options(port, options): options.configuration = port.default_configuration() if options.pixel_tests is None: - options.pixel_tests = True + options.pixel_tests = port.default_pixel_tests() if not options.time_out_ms: - if options.configuration == "Debug": - options.time_out_ms = str(2 * port.default_test_timeout_ms()) - else: - options.time_out_ms = str(port.default_test_timeout_ms()) + options.time_out_ms = str(port.default_timeout_ms()) options.slow_time_out_ms = str(5 * int(options.time_out_ms)) @@ -436,8 +447,8 @@ def parse_args(args=None): return option_parser.parse_args(args) -def main(): - options, args = parse_args() +def main(argv=None): + options, args = parse_args(argv) if options.platform and 'test' in options.platform: # It's a bit lame to import mocks into real code, but this allows the user # to run tests against the test platform interactively, which is useful for @@ -446,7 +457,14 @@ def main(): host = MockHost() else: host = Host() - port = host.port_factory.get(options.platform, options) + + try: + port = host.port_factory.get(options.platform, options) + except NotImplementedError, e: + # FIXME: is this the best way to handle unsupported port names? + print >> sys.stderr, str(e) + return EXCEPTIONAL_EXIT_STATUS + logging.getLogger().setLevel(logging.DEBUG if options.verbose else logging.INFO) return run(port, options, args) @@ -454,12 +472,7 @@ def main(): if '__main__' == __name__: try: sys.exit(main()) - except KeyboardInterrupt: - # This mirrors what the shell normally does. - INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128 - sys.exit(INTERRUPTED_EXIT_STATUS) - except WorkerException: - # This is a randomly chosen exit code that can be tested against to - # indicate that an unexpected exception occurred. - EXCEPTIONAL_EXIT_STATUS = 254 + except Exception, e: + if e.__class__ in (KeyboardInterrupt, TestRunInterruptedException): + sys.exit(INTERRUPTED_EXIT_STATUS) sys.exit(EXCEPTIONAL_EXIT_STATUS) diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py index d5c1e14e1..c106fbf47 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py @@ -46,10 +46,12 @@ import unittest from webkitpy.common.system import outputcapture, path from webkitpy.common.system.crashlogs_unittest import make_mock_crash_report_darwin from webkitpy.common.system.systemhost import SystemHost +from webkitpy.common.host import Host from webkitpy.common.host_mock import MockHost from webkitpy.layout_tests import port from webkitpy.layout_tests import run_webkit_tests +from webkitpy.layout_tests.controllers.manager_worker_broker import WorkerException from webkitpy.layout_tests.port import Port from webkitpy.layout_tests.port.test import TestPort, TestDriver from webkitpy.test.skip import skip_if @@ -81,18 +83,22 @@ def parse_args(extra_args=None, record_results=False, tests_included=False, new_ return run_webkit_tests.parse_args(args) -def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None): +def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None, shared_port=True): options, parsed_args = parse_args(extra_args, record_results, tests_included) if not port_obj: host = host or MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) + + if shared_port: + port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj + buildbot_output = StringIO.StringIO() regular_output = StringIO.StringIO() res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output) return res == 0 and not regular_output.getvalue() and not buildbot_output.getvalue() -def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None, new_results=False): +def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None, new_results=False, shared_port=True): options, parsed_args = parse_args(extra_args=extra_args, record_results=record_results, tests_included=tests_included, @@ -101,11 +107,13 @@ def logging_run(extra_args=None, port_obj=None, record_results=False, tests_incl if not port_obj: port_obj = host.port_factory.get(port_name=options.platform, options=options) - res, buildbot_output, regular_output = run_and_capture(port_obj, options, parsed_args) + res, buildbot_output, regular_output = run_and_capture(port_obj, options, parsed_args, shared_port) return (res, buildbot_output, regular_output, host.user) -def run_and_capture(port_obj, options, parsed_args): +def run_and_capture(port_obj, options, parsed_args, shared_port=True): + if shared_port: + port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj oc = outputcapture.OutputCapture() try: oc.capture_output() @@ -302,14 +310,14 @@ class MainTest(unittest.TestCase, StreamTestingMixin): def test_child_processes_2(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( - ['--print', 'config', '--child-processes', '2']) + ['--print', 'config', '--child-processes', '2'], shared_port=False) self.assertTrue(any(['Running 2 ' in line for line in regular_output.buflist])) def test_child_processes_min(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( ['--print', 'config', '--child-processes', '2', 'passes'], - tests_included=True) + tests_included=True, shared_port=False) self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist])) def test_dryrun(self): @@ -333,8 +341,8 @@ class MainTest(unittest.TestCase, StreamTestingMixin): ['failures/expected/exception.html', '--child-processes', '1'], tests_included=True) if self.should_test_processes: - self.assertRaises(run_webkit_tests.WorkerException, logging_run, - ['--child-processes', '2', '--force', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True) + self.assertRaises(WorkerException, logging_run, + ['--child-processes', '2', '--force', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False) def test_full_results_html(self): # FIXME: verify html? @@ -364,7 +372,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): if self.should_test_processes: self.assertRaises(KeyboardInterrupt, logging_run, - ['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--force'], tests_included=True) + ['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--force'], tests_included=True, shared_port=False) def test_no_tests_found(self): res, out, err, user = logging_run(['resources'], tests_included=True) @@ -754,6 +762,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): # Now we test that --clobber-old-results does remove the old entries and the old retries, # and that we don't retry again. + host = MockHost() res, out, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host) self.assertEquals(res, 1) self.assertTrue('Clobbering old results' in err.getvalue()) @@ -763,22 +772,16 @@ class MainTest(unittest.TestCase, StreamTestingMixin): self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt')) self.assertFalse(host.filesystem.exists('retries')) - - # These next tests test that we run the tests in ascending alphabetical - # order per directory. HTTP tests are sharded separately from other tests, - # so we have to test both. - def assert_run_order(self, child_processes='1'): - tests_run = get_tests_run(['--child-processes', child_processes, 'passes'], - tests_included=True, flatten_batches=True) + def test_run_order__inline(self): + # These next tests test that we run the tests in ascending alphabetical + # order per directory. HTTP tests are sharded separately from other tests, + # so we have to test both. + tests_run = get_tests_run(['passes'], tests_included=True, flatten_batches=True) self.assertEquals(tests_run, sorted(tests_run)) - tests_run = get_tests_run(['--child-processes', child_processes, 'http/tests/passes'], - tests_included=True, flatten_batches=True) + tests_run = get_tests_run(['http/tests/passes'], tests_included=True, flatten_batches=True) self.assertEquals(tests_run, sorted(tests_run)) - def test_run_order__inline(self): - self.assert_run_order() - def test_tolerance(self): class ImageDiffTestPort(TestPort): def diff_image(self, expected_contents, actual_contents, tolerance=None): @@ -901,6 +904,20 @@ class MainTest(unittest.TestCase, StreamTestingMixin): self.assertEquals(full_results['has_wdiff'], False) self.assertEquals(full_results['has_pretty_patch'], False) + def test_unsupported_platform(self): + oc = outputcapture.OutputCapture() + try: + oc.capture_output() + res = run_webkit_tests.main(['--platform', 'foo']) + finally: + stdout, stderr, logs = oc.restore_output() + + self.assertEquals(res, run_webkit_tests.EXCEPTIONAL_EXIT_STATUS) + self.assertEquals(stdout, '') + self.assertTrue('unsupported platform' in stderr) + + # This is empty because we don't even get a chance to configure the logger before failing. + self.assertEquals(logs, '') class EndToEndTest(unittest.TestCase): def parse_full_results(self, full_results_text): @@ -1009,5 +1026,21 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin): "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err) +class PortTest(unittest.TestCase): + def assert_mock_port_works(self, port_name, args=[]): + self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name, 'fast/harness/results.html'], tests_included=True, host=Host())) + + def disabled_test_chromium_mac_lion(self): + self.assert_mock_port_works('chromium-mac-lion') + + def disabled_test_chromium_mac_lion_in_test_shell_mode(self): + self.assert_mock_port_works('chromium-mac-lion', args=['--additional-drt-flag=--test-shell']) + + def disabled_test_qt_linux(self): + self.assert_mock_port_works('qt-linux') + + def disabled_test_mac_lion(self): + self.assert_mock_port_works('mac-lion') + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/views/metered_stream.py b/Tools/Scripts/webkitpy/layout_tests/views/metered_stream.py index 814437780..ec73a93ac 100644 --- a/Tools/Scripts/webkitpy/layout_tests/views/metered_stream.py +++ b/Tools/Scripts/webkitpy/layout_tests/views/metered_stream.py @@ -58,7 +58,6 @@ class MeteredStream(object): self._verbose = verbose self._time_fn = time_fn or time.time self._pid = pid or os.getpid() - self._isatty = self._stream.isatty() self._erasing = self._isatty and not verbose self._last_partial_line = '' @@ -128,4 +127,4 @@ class _LogHandler(logging.Handler): self.name = LOG_HANDLER_NAME def emit(self, record): - self._meter.writeln(record.getMessage(), record.created) + self._meter.writeln(record.getMessage(), record.created, record.process) diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest.py b/Tools/Scripts/webkitpy/performance_tests/perftest.py index f76d85b1b..de63f3e8d 100644 --- a/Tools/Scripts/webkitpy/performance_tests/perftest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftest.py @@ -40,7 +40,7 @@ import sys import time # Import for auto-install -if sys.platform != 'win32': +if sys.platform not in ('cygwin', 'win32'): # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279. import webkitpy.thirdparty.autoinstalled.webpagereplay.replay diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py index 9a3757128..ab4386443 100644..100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py @@ -85,6 +85,8 @@ class PerfTestsRunner(object): help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), + optparse.make_option("--no-build", dest="build", action="store_false", + help="Don't check to see if the DumpRenderTree build is up-to-date."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, @@ -101,6 +103,8 @@ class PerfTestsRunner(object): help="Use WebKitTestRunner rather than DumpRenderTree."), optparse.make_option("--replay", dest="replay", action="store_true", default=False, help="Run replay tests."), + optparse.make_option("--force", dest="skipped", action="store_true", default=False, + help="Run all tests, including the ones in the Skipped list."), ] return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) @@ -128,7 +132,7 @@ class PerfTestsRunner(object): tests = [] for path in test_files: relative_path = self._port.relative_perf_test_filename(path).replace('\\', '/') - if self._port.skips_perf_test(relative_path): + if self._port.skips_perf_test(relative_path) and not self._options.skipped: continue test = PerfTestFactory.create_perf_test(self._port, relative_path, path) tests.append(test) diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py index 8e1eb57ff..de3528cb1 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py @@ -397,6 +397,17 @@ max 1120 port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported'] self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html']) + def test_collect_tests_with_skipped_list(self): + runner, port = self.create_runner(args=['--force']) + + self._add_file(runner, 'inspector', 'test1.html') + self._add_file(runner, 'inspector', 'unsupported_test1.html') + self._add_file(runner, 'inspector', 'test2.html') + self._add_file(runner, 'inspector/resources', 'resource_file.html') + self._add_file(runner, 'unsupported', 'unsupported_test2.html') + port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported'] + self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html', 'inspector/unsupported_test1.html', 'unsupported/unsupported_test2.html']) + def test_collect_tests_with_page_load_svg(self): runner, port = self.create_runner() self._add_file(runner, 'PageLoad', 'some-svg-test.svg') diff --git a/Tools/Scripts/webkitpy/pylintrc b/Tools/Scripts/webkitpy/pylintrc new file mode 100644 index 000000000..dae778d63 --- /dev/null +++ b/Tools/Scripts/webkitpy/pylintrc @@ -0,0 +1,309 @@ +# Copyright (c) 2012 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +# CHANGED: +# C0103: Invalid name "" +# C0111: Missing docstring +# C0302: Too many lines in module (N) +# I0010: Unable to consider inline option '' +# I0011: Locally disabling WNNNN +# +# R0201: Method could be a function +# R0801: Similar lines in N files +# R0901: Too many ancestors (8/7) +# R0902: Too many instance attributes (N/7) +# R0903: Too few public methods (N/2) +# R0904: Too many public methods (N/20) +# R0911: Too many return statements (N/6) +# R0912: Too many branches (N/12) +# R0913: Too many arguments (N/5) +# R0914: Too many local variables (N/15) +# R0915: Too many statements (N/50) +# R0921: Abstract class not referenced +# R0922: Abstract class is only referenced 1 times +# W0122: Use of the exec statement +# W0141: Used builtin function '' +# W0212: Access to a protected member X of a client class +# W0142: Used * or ** magic +# W0402: Uses of a deprecated module 'string' +# W0404: 41: Reimport 'XX' (imported line NN) +# W0511: TODO +# W0603: Using the global statement +# W0703: Catch "Exception" +# W1201: Specify string format arguments as logging function parameters +disable=C0103,C0111,C0302,I0010,I0011,R0201,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,W0122,W0141,W0142,W0212,W0402,W0404,W0511,W0603,W0703,W1201 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=yes + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +# CHANGED: +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=200 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +# CHANGED: +indent-string=' ' + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/Tools/Scripts/webkitpy/style/checker.py b/Tools/Scripts/webkitpy/style/checker.py index dff790ec1..8cd8745bf 100644 --- a/Tools/Scripts/webkitpy/style/checker.py +++ b/Tools/Scripts/webkitpy/style/checker.py @@ -152,7 +152,8 @@ _PATH_RULES_SPECIFIER = [ "-whitespace/declaration"]), ([# Qt's MiniBrowser has no config.h - "Tools/MiniBrowser/qt"], + "Tools/MiniBrowser/qt", + "Tools/MiniBrowser/qt/raw"], ["-build/include"]), ([# The Qt APIs use Qt/QML naming style, which includes diff --git a/Tools/Scripts/webkitpy/style/checkers/png.py b/Tools/Scripts/webkitpy/style/checkers/png.py index 30b7a1439..430d6f0a0 100644 --- a/Tools/Scripts/webkitpy/style/checkers/png.py +++ b/Tools/Scripts/webkitpy/style/checkers/png.py @@ -27,6 +27,7 @@ import os import re +from webkitpy.common import checksvnconfigfile from webkitpy.common import read_checksum_from_png from webkitpy.common.system.systemhost import SystemHost from webkitpy.common.checkout.scm.detection import SCMDetector @@ -54,49 +55,21 @@ class PNGChecker(object): self._handle_style_error(0, 'image/png', 5, "Image lacks a checksum. Generate pngs using run-webkit-tests to ensure they have a checksum.") if detection == "git": - config_file_path = self._config_file_path() - there_is_enable_line = False - there_is_png_line = False - - try: - config_file = self._fs.read_text_file(config_file_path) - except IOError: - errorstr = "There is no " + config_file_path - self._handle_style_error(0, 'image/png', 5, errorstr) - return - - errorstr_autoprop = "Have to enable auto props in the subversion config file (" + config_file_path + " \"enable-auto-props = yes\"). " - errorstr_png = "Have to set the svn:mime-type in the subversion config file (" + config_file_path + " \"*.png = svn:mime-type=image/png\")." - - for line in config_file.split('\n'): - if not there_is_enable_line: - match = re.search("^\s*enable-auto-props\s*=\s*yes", line) - if match: - there_is_enable_line = True - errorstr_autoprop = "" - continue - - if not there_is_png_line: - match = re.search("^\s*\*\.png\s*=\s*svn:mime-type=image/png", line) - if match: - there_is_png_line = True - errorstr_png = "" - continue - - errorstr = errorstr_autoprop + errorstr_png - if errorstr: - self._handle_style_error(0, 'image/png', 5, errorstr) + (file_missing, autoprop_missing, png_missing) = checksvnconfigfile.check(self._host, self._fs) + config_file_path = checksvnconfigfile.config_file_path(self._host, self._fs) + + if file_missing: + self._handle_style_error(0, 'image/png', 5, "There is no SVN config file. (%s)" % config_file_path) + elif autoprop_missing and png_missing: + self._handle_style_error(0, 'image/png', 5, checksvnconfigfile.errorstr_autoprop(config_file_path) + checksvnconfigfile.errorstr_png(config_file_path)) + elif autoprop_missing: + self._handle_style_error(0, 'image/png', 5, checksvnconfigfile.errorstr_autoprop(config_file_path)) + elif png_missing: + self._handle_style_error(0, 'image/png', 5, checksvnconfigfile.errorstr_png(config_file_path)) elif detection == "svn": prop_get = self._detector.propget("svn:mime-type", self._file_path) if prop_get != "image/png": - errorstr = "Set the svn:mime-type property (svn propset svn:mime-type image/png " + self._file_path + ")." + errorstr = "Set the svn:mime-type property (svn propset svn:mime-type image/png %s)." % self._file_path self._handle_style_error(0, 'image/png', 5, errorstr) - def _config_file_path(self): - config_file = "" - if self._host.platform.is_win(): - config_file_path = self._fs.join(os.environ['APPDATA'], "Subversion\config") - else: - config_file_path = self._fs.join(self._fs.expanduser("~"), ".subversion/config") - return config_file_path diff --git a/Tools/Scripts/webkitpy/thirdparty/__init__.py b/Tools/Scripts/webkitpy/thirdparty/__init__.py index 078e18041..b1edd4d0b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/__init__.py +++ b/Tools/Scripts/webkitpy/thirdparty/__init__.py @@ -74,6 +74,8 @@ class AutoinstallImportHook(object): self._install_mechanize() elif '.pep8' in fullname: self._install_pep8() + elif '.pylint' in fullname: + self._install_pylint() elif '.coverage' in fullname: self._install_coverage() elif '.eliza' in fullname: @@ -93,6 +95,12 @@ class AutoinstallImportHook(object): self._install("http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b", "pep8-0.5.0/pep8.py") + def _install_pylint(self): + if not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "pylint")): + self._install('http://pypi.python.org/packages/source/p/pylint/pylint-0.25.1.tar.gz#md5=728bbc2b339bc3749af013709a7f87a5', 'pylint-0.25.1') + self._fs.move(self._fs.join(_AUTOINSTALLED_DIR, "pylint-0.25.1"), self._fs.join(_AUTOINSTALLED_DIR, "pylint")) + + # autoinstalled.buildbot is used by BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py # and should ideally match the version of BuildBot used at build.webkit.org. def _install_buildbot(self): diff --git a/Tools/Scripts/webkitpy/tool/commands/download.py b/Tools/Scripts/webkitpy/tool/commands/download.py index 4f6b7370e..60c89208f 100644 --- a/Tools/Scripts/webkitpy/tool/commands/download.py +++ b/Tools/Scripts/webkitpy/tool/commands/download.py @@ -90,6 +90,7 @@ class Land(AbstractSequencedCommand): argument_names = "[BUGID]" show_in_main_help = True steps = [ + steps.AddSvnMimetypeForPng, steps.UpdateChangeLogsWithReviewer, steps.ValidateReviewer, steps.ValidateChangeLogs, # We do this after UpdateChangeLogsWithReviewer to avoid not having to cache the diff twice. diff --git a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py index 7f19fbd7a..79b729aad 100644 --- a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -136,7 +136,7 @@ MockWatchList: determine_cc_and_messages def test_land_cowboy(self): expected_stderr = """MOCK run_and_throw_if_fail: ['mock-prepare-ChangeLog', '--email=MOCK email', '--merge-base=None', 'MockFile1'], cwd=/mock-checkout MOCK run_and_throw_if_fail: ['mock-check-webkit-style', '--git-commit', 'MOCK git commit', '--diff-files', 'MockFile1', '--filter', '-changelog'], cwd=/mock-checkout -MOCK run_command: ['ruby', '-I', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch/prettify.rb'], cwd=None +MOCK run_command: ['ruby', '-I', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch/prettify.rb'], cwd=None, input=Patch1 MOCK: user.open_url: file://... Was that diff correct? Building WebKit diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py index bd890cc8e..cb7254ba2 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py @@ -32,6 +32,7 @@ import optparse import os.path import re import shutil +import sys import urllib import webkitpy.common.config.urls as config_urls @@ -67,26 +68,27 @@ class AbstractRebaseliningCommand(AbstractDeclarativeCommand): class RebaselineTest(AbstractRebaseliningCommand): - name = "rebaseline-test" - help_text = "Rebaseline a single test from a buildbot. (Currently works only with build.chromium.org buildbots.)" - argument_names = "BUILDER_NAME TEST_NAME [PLATFORMS_TO_MOVE_EXISTING_BASELINES_TO]" + name = "rebaseline-test-internal" + help_text = "Rebaseline a single test from a buildbot. Only intended for use by other webkit-patch commands." def __init__(self): options = [ - optparse.make_option("--print-scm-changes", action="store_true", help="Print modifcations to the scm (as a json dict) rather than actually modifying the scm"), + optparse.make_option("--builder", help="Builder to pull new baselines from"), + optparse.make_option("--platform-to-move-to", help="Platform to move existing baselines to before rebaselining. This is for dealing with bringing up new ports that interact with non-tree portions of the fallback graph."), + optparse.make_option("--test", help="Test to rebaseline"), ] AbstractRebaseliningCommand.__init__(self, options=options) - self._print_scm_changes = False - self._scm_changes = {} + self._scm_changes = {'add': []} def _results_url(self, builder_name): - # FIXME: Generalize this command to work with non-build.chromium.org builders. - builder = self._tool.chromium_buildbot().builder_with_name(builder_name) - return builder.accumulated_results_url() + return self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name).latest_layout_test_results_url() def _baseline_directory(self, builder_name): port = self._tool.port_factory.get_from_builder_name(builder_name) - return port.baseline_path() + override_dir = builders.rebaseline_override_dir(builder_name) + if override_dir: + return self._tool.filesystem.join(port.layout_tests_dir(), 'platform', override_dir) + return port.baseline_version_dir() def _copy_existing_baseline(self, platforms_to_move_existing_baselines_to, test_name, suffix): old_baselines = [] @@ -129,10 +131,7 @@ class RebaselineTest(AbstractRebaseliningCommand): self._add_to_scm(target_baseline) def _add_to_scm(self, path): - if self._print_scm_changes: - self._scm_changes['add'].append(path) - else: - self._tool.scm().add(path) + self._scm_changes['add'].append(path) def _update_expectations_file(self, builder_name, test_name): port = self._tool.port_factory.get_from_builder_name(builder_name) @@ -173,16 +172,8 @@ class RebaselineTest(AbstractRebaseliningCommand): def execute(self, options, args, tool): self._baseline_suffix_list = options.suffixes.split(',') - self._print_scm_changes = options.print_scm_changes - self._scm_changes = {'add': [], 'delete': []} - - if len(args) > 2: - platforms_to_move_existing_baselines_to = args[2:] - else: - platforms_to_move_existing_baselines_to = None - self._rebaseline_test_and_update_expectations(args[0], args[1], platforms_to_move_existing_baselines_to) - if self._print_scm_changes: - print json.dumps(self._scm_changes) + self._rebaseline_test_and_update_expectations(options.builder, options.test, options.platform_to_move_to) + print json.dumps(self._scm_changes) class OptimizeBaselines(AbstractRebaseliningCommand): @@ -233,32 +224,103 @@ class AnalyzeBaselines(AbstractRebaseliningCommand): self._analyze_baseline(test_name) -class RebaselineExpectations(AbstractDeclarativeCommand): - name = "rebaseline-expectations" - help_text = "Rebaselines the tests indicated in TestExpectations." - - def __init__(self): - options = [ +class AbstractParallelRebaselineCommand(AbstractDeclarativeCommand): + def __init__(self, options=None): + options = options or [] + options.extend([ optparse.make_option('--no-optimize', dest='optimize', action='store_false', default=True, help=('Do not optimize/de-dup the expectations after rebaselining ' '(default is to de-dup automatically). ' - 'You can use "webkit-patch optimize-baselines" to optimize separately.')), - ] + 'You can use "webkit-patch optimize-baselines" to optimize separately.'))]) AbstractDeclarativeCommand.__init__(self, options=options) def _run_webkit_patch(self, args): try: self._tool.executive.run_command([self._tool.path()] + args, cwd=self._tool.scm().checkout_root) except ScriptError, e: - pass + _log.error(e) + + def _builders_to_fetch_from(self, builders): + # This routine returns the subset of builders that will cover all of the baseline search paths + # used in the input list. In particular, if the input list contains both Release and Debug + # versions of a configuration, we *only* return the Release version (since we don't save + # debug versions of baselines). + release_builders = set() + debug_builders = set() + builders_to_fallback_paths = {} + for builder in builders: + port = self._tool.port_factory.get_from_builder_name(builder) + if port.test_configuration().build_type == 'Release': + release_builders.add(builder) + else: + debug_builders.add(builder) + for builder in list(release_builders) + list(debug_builders): + port = self._tool.port_factory.get_from_builder_name(builder) + fallback_path = port.baseline_search_path() + if fallback_path not in builders_to_fallback_paths.values(): + builders_to_fallback_paths[builder] = fallback_path + return builders_to_fallback_paths.keys() + + def _rebaseline_commands(self, test_list): + path_to_webkit_patch = self._tool.path() + cwd = self._tool.scm().checkout_root + commands = [] + for test in test_list: + for builder in self._builders_to_fetch_from(test_list[test]): + suffixes = ','.join(test_list[test][builder]) + cmd_line = [path_to_webkit_patch, 'rebaseline-test-internal', '--suffixes', suffixes, '--builder', builder, '--test', test] + commands.append(tuple([cmd_line, cwd])) + return commands + + def _files_to_add(self, command_results): + files_to_add = set() + for output in [result[1].split('\n') for result in command_results]: + file_added = False + for line in output: + try: + files_to_add.update(json.loads(line)['add']) + file_added = True + except ValueError, e: + _log.debug('"%s" is not a JSON object, ignoring' % line) + + if not file_added: + _log.debug('Could not add file based off output "%s"' % output) + + + return list(files_to_add) + + def _optimize_baselines(self, test_list): + # We don't run this in parallel because modifying the SCM in parallel is unreliable. + for test in test_list: + all_suffixes = set() + for builder in self._builders_to_fetch_from(test_list[test]): + all_suffixes.update(test_list[test][builder]) + self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test]) + + def _rebaseline(self, options, test_list): + commands = self._rebaseline_commands(test_list) + command_results = self._tool.executive.run_in_parallel(commands) + + files_to_add = self._files_to_add(command_results) + self._tool.scm().add_list(list(files_to_add)) + + if options.optimize: + self._optimize_baselines(test_list) + + +class RebaselineJson(AbstractParallelRebaselineCommand): + name = "rebaseline-json" + help_text = "Rebaseline based off JSON passed to stdin. Intended to only be called from other scripts." - def _is_supported_port(self, port_name): - # FIXME: Support non-Chromium ports. - return port_name.startswith('chromium-') + def execute(self, options, args, tool): + self._rebaseline(options, json.loads(sys.stdin.read())) + + +class RebaselineExpectations(AbstractParallelRebaselineCommand): + name = "rebaseline-expectations" + help_text = "Rebaselines the tests indicated in TestExpectations." def _update_expectations_file(self, port_name): - if not self._is_supported_port(port_name): - return port = self._tool.port_factory.get(port_name) # FIXME: This will intentionally skip over any REBASELINE expectations that were in an overrides file. @@ -276,71 +338,89 @@ class RebaselineExpectations(AbstractDeclarativeCommand): tests_to_rebaseline[test] = suffixes_for_expectations(expectations.get_expectations(test)) return tests_to_rebaseline - def _rebaseline_port(self, port_name): - if not self._is_supported_port(port_name): - return + def _add_tests_to_rebaseline_for_port(self, port_name): builder_name = builders.builder_name_for_port_name(port_name) if not builder_name: return - _log.info("Retrieving results for %s from %s." % (port_name, builder_name)) - for test_name, suffixes in self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).iteritems(): - self._touched_tests.setdefault(test_name, set()).update(set(suffixes)) + tests = self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).items() + + if tests: + _log.info("Retrieving results for %s from %s." % (port_name, builder_name)) + + for test_name, suffixes in tests: _log.info(" %s (%s)" % (test_name, ','.join(suffixes))) - # FIXME: we should use executive.run_in_parallel() to speed this up. - self._run_webkit_patch(['rebaseline-test', '--suffixes', ','.join(suffixes), builder_name, test_name]) + if test_name not in self._test_list: + self._test_list[test_name] = {} + self._test_list[test_name][builder_name] = suffixes def execute(self, options, args, tool): - self._touched_tests = {} + self._test_list = {} for port_name in tool.port_factory.all_port_names(): - self._rebaseline_port(port_name) + self._add_tests_to_rebaseline_for_port(port_name) + self._rebaseline(options, self._test_list) + for port_name in tool.port_factory.all_port_names(): self._update_expectations_file(port_name) - if not options.optimize: - return - for test_name, suffixes in self._touched_tests.iteritems(): - _log.info("Optimizing baselines for %s (%s)." % (test_name, ','.join(suffixes))) - self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(suffixes), test_name]) -class Rebaseline(AbstractDeclarativeCommand): +class Rebaseline(AbstractParallelRebaselineCommand): name = "rebaseline" - help_text = "Replaces local expected.txt files with new results from build bots" - - # FIXME: This should share more code with FailureReason._builder_to_explain - def _builder_to_pull_from(self): - builder_statuses = self._tool.buildbot.builder_statuses() - red_statuses = [status for status in builder_statuses if not status["is_green"]] - _log.info("%s failing" % (pluralize("builder", len(red_statuses)))) - builder_choices = [status["name"] for status in red_statuses] - chosen_name = self._tool.user.prompt_with_list("Which builder to pull results from:", builder_choices) - # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. - for status in red_statuses: - if status["name"] == chosen_name: - return (self._tool.buildbot.builder_with_name(chosen_name), status["build_number"]) - - def _replace_expectation_with_remote_result(self, local_file, remote_file): - (downloaded_file, headers) = urllib.urlretrieve(remote_file) - shutil.move(downloaded_file, local_file) - - def _tests_to_update(self, build): - failing_tests = build.layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch]) - return self._tool.user.prompt_with_list("Which test(s) to rebaseline:", failing_tests, can_choose_multiple=True) - - def _results_url_for_test(self, build, test): - test_base = os.path.splitext(test)[0] - actual_path = test_base + "-actual.txt" - return build.results_url() + "/" + actual_path + help_text = "Rebaseline tests with results from the build bots. Shows the list of failing tests on the builders if no test names are provided." + argument_names = "[TEST_NAMES]" + + def __init__(self): + options = [ + optparse.make_option("--builders", default=None, action="append", help="Comma-separated-list of builders to pull new baselines from (can also be provided multiple times)"), + optparse.make_option("--suffixes", default=BASELINE_SUFFIX_LIST, action="append", help="Comma-separated-list of file types to rebaseline (can also be provided multiple times)"), + ] + AbstractParallelRebaselineCommand.__init__(self, options=options) + + def _builders_to_pull_from(self): + chromium_buildbot_builder_names = [] + webkit_buildbot_builder_names = [] + for name in builders.all_builder_names(): + if self._tool.port_factory.get_from_builder_name(name).is_chromium(): + chromium_buildbot_builder_names.append(name) + else: + webkit_buildbot_builder_names.append(name) + + titles = ["build.webkit.org bots", "build.chromium.org bots"] + lists = [webkit_buildbot_builder_names, chromium_buildbot_builder_names] + + chosen_names = self._tool.user.prompt_with_multiple_lists("Which builder to pull results from:", titles, lists, can_choose_multiple=True) + return [self._builder_with_name(name) for name in chosen_names] + + def _builder_with_name(self, name): + return self._tool.buildbot_for_builder_name(name).builder_with_name(name) + + def _tests_to_update(self, builder): + failing_tests = builder.latest_layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch]) + return self._tool.user.prompt_with_list("Which test(s) to rebaseline for %s:" % builder.name(), failing_tests, can_choose_multiple=True) + + def _suffixes_to_update(self, options): + suffixes = [] + for suffix_list in options.suffixes: + suffixes += suffix_list.split(",") + return suffixes def execute(self, options, args, tool): - builder, build_number = self._builder_to_pull_from() - build = builder.build(build_number) - port = tool.port_factory.get_from_builder_name(builder.name()) - - for test in self._tests_to_update(build): - results_url = self._results_url_for_test(build, test) - # Port operates with absolute paths. - expected_file = port.expected_filename(test, '.txt') - _log.info(test) - self._replace_expectation_with_remote_result(expected_file, results_url) - - # FIXME: We should handle new results too. + if options.builders: + builders = [] + for builder_names in options.builders: + builders += [self._builder_with_name(name) for name in builder_names.split(",")] + else: + builders = self._builders_to_pull_from() + + test_list = {} + + for builder in builders: + tests = args or self._tests_to_update(builder) + for test in tests: + if test not in test_list: + test_list[test] = {} + test_list[test][builder.name()] = self._suffixes_to_update(options) + + if options.verbose: + print "rebaseline-json: " + str(test_list) + + self._rebaseline(options, test_list) diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py index 84036d46e..a3b9efaeb 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -32,6 +32,7 @@ from webkitpy.common.system.outputcapture import OutputCapture from webkitpy.thirdparty.mock import Mock from webkitpy.tool.commands.rebaseline import * from webkitpy.tool.mocktool import MockTool, MockOptions +from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder from webkitpy.common.system.executive_mock import MockExecutive @@ -42,6 +43,20 @@ class TestRebaseline(unittest.TestCase): build = Mock() OutputCapture().assert_outputs(self, command._tests_to_update, [build]) + def test_baseline_directory(self): + command = RebaselineTest() + tool = MockTool() + command.bind_to_tool(tool) + self.assertEqual(command._baseline_directory("Apple Win XP Debug (Tests)"), "/mock-checkout/LayoutTests/platform/win-xp") + self.assertEqual(command._baseline_directory("Apple Win 7 Release (Tests)"), "/mock-checkout/LayoutTests/platform/win") + self.assertEqual(command._baseline_directory("Apple Lion Release WK1 (Tests)"), "/mock-checkout/LayoutTests/platform/mac") + self.assertEqual(command._baseline_directory("Apple Lion Release WK2 (Tests)"), "/mock-checkout/LayoutTests/platform/mac-wk2") + self.assertEqual(command._baseline_directory("GTK Linux 32-bit Release"), "/mock-checkout/LayoutTests/platform/gtk") + self.assertEqual(command._baseline_directory("EFL Linux 64-bit Debug"), "/mock-checkout/LayoutTests/platform/efl") + self.assertEqual(command._baseline_directory("Qt Linux Release"), "/mock-checkout/LayoutTests/platform/qt") + self.assertEqual(command._baseline_directory("Webkit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac") + self.assertEqual(command._baseline_directory("Webkit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") + def test_rebaseline_updates_expectations_file_noop(self): command = RebaselineTest() tool = MockTool() @@ -60,6 +75,7 @@ BUGA DEBUG : fast/css/large-list-of-rules-crash.html = TEXT expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. """ OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) @@ -85,7 +101,7 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "BUGX LEOPARD SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html = IMAGE\n") + self.assertEqual(new_expectations, "BUGX SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html = IMAGE\n") def test_rebaseline_does_not_include_overrides(self): command = RebaselineTest() @@ -105,7 +121,7 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "BUGX LEOPARD SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html = IMAGE\n") + self.assertEqual(new_expectations, "BUGX SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html = IMAGE\n") def test_rebaseline_test(self): command = RebaselineTest() @@ -155,10 +171,9 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu tool.filesystem.write_text_file(os.path.join(lion_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Copying baseline from /mock-checkout/LayoutTests/platform/chromium-mac/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt. -Copying baseline from /mock-checkout/LayoutTests/platform/chromium-mac/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-leopard/userscripts/another-test-expected.txt. Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard", "chromium-mac-leopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_no_overwrite_test(self): command = RebaselineTest() @@ -176,6 +191,37 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu """ OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + def test_rebaseline_all(self): + old_exact_matches = builders._exact_matches + builders._exact_matches = { + "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, + "MOCK builder (Debug)": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier", "debug"])}, + } + + command = RebaselineJson() + tool = MockTool() + options = MockOptions() + options.optimize = True + command.bind_to_tool(tool) + tool.executive = MockExecutive(should_log=True) + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html'], cwd=/mock-checkout +""" + OutputCapture().assert_outputs(self, command._rebaseline, [options, {"user-scripts/another-test.html":{"MOCK builder": ["txt", "png"]}}], expected_stderr=expected_stderr) + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html'], cwd=/mock-checkout +""" + OutputCapture().assert_outputs(self, command._rebaseline, [options, {"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt", "png"]}}], expected_stderr=expected_stderr) + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'user-scripts/another-test.html'], cwd=/mock-checkout +""" + OutputCapture().assert_outputs(self, command._rebaseline, [options, {"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt", "png"], "MOCK builder": ["txt"]}}], expected_stderr=expected_stderr) + + builders._exact_matches = old_exact_matches + def test_rebaseline_expectations(self): command = RebaselineExpectations() tool = MockTool() @@ -189,15 +235,18 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu # Don't enable logging until after we create the mock expectation files as some Port.__init__'s run subcommands. tool.executive = MockExecutive(should_log=True) + def run_in_parallel(commands): + print commands + return "" + + tool.executive.run_in_parallel = run_in_parallel + expected_logs = """Retrieving results for chromium-linux-x86 from Webkit Linux 32. userscripts/another-test.html (txt) userscripts/images.svg (png) Retrieving results for chromium-linux-x86_64 from Webkit Linux. userscripts/another-test.html (txt) userscripts/images.svg (png) -Retrieving results for chromium-mac-leopard from Webkit Mac10.5. - userscripts/another-test.html (txt) - userscripts/images.svg (png) Retrieving results for chromium-mac-lion from Webkit Mac10.7. userscripts/another-test.html (txt) userscripts/images.svg (png) @@ -210,36 +259,57 @@ Retrieving results for chromium-win-win7 from Webkit Win7. Retrieving results for chromium-win-xp from Webkit Win. userscripts/another-test.html (txt) userscripts/images.svg (png) +Retrieving results for efl from EFL Linux 64-bit Release. + userscripts/another-test.html (txt) + userscripts/images.svg (png) +Retrieving results for gtk from GTK Linux 64-bit Release. + userscripts/another-test.html (txt) + userscripts/images.svg (png) +Retrieving results for mac-lion from Apple Lion Release WK1 (Tests). + userscripts/another-test.html (txt) + userscripts/images.svg (png) +Retrieving results for qt-linux from Qt Linux Release. + userscripts/another-test.html (txt) + userscripts/images.svg (png) +Retrieving results for win-7sp0 from Apple Win 7 Release (Tests). + userscripts/another-test.html (txt) + userscripts/images.svg (png) +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. +Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky. """ - expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Linux 32', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Linux 32', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Linux', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Linux', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.5', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.5', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.7', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.7', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.6', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.6', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Win7', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Win7', 'userscripts/images.svg'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Win', 'userscripts/another-test.html'], cwd=/mock-checkout -MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Win', 'userscripts/images.svg'], cwd=/mock-checkout + expected_stdout = """[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')] """ - command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])} - OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=False), [], tool], expected_logs=expected_logs, expected_stderr=expected_stderr) + expected_stderr = """MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +""" - expected_logs_with_optimize = expected_logs + ( - "Optimizing baselines for userscripts/another-test.html (txt).\n" - "Optimizing baselines for userscripts/images.svg (png).\n") - expected_stderr_with_optimize = expected_stderr + ( - "MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'userscripts/another-test.html'], cwd=/mock-checkout\n" - "MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'png', 'userscripts/images.svg'], cwd=/mock-checkout\n") + command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])} + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=False), [], tool], expected_logs=expected_logs, expected_stdout=expected_stdout, expected_stderr=expected_stderr) + + expected_stderr_with_optimize = """MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'userscripts/another-test.html'], cwd=/mock-checkout +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'png', 'userscripts/images.svg'], cwd=/mock-checkout +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +MOCK run_command: ['qmake', '-v'], cwd=None +""" command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])} - OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True), [], tool], expected_logs=expected_logs_with_optimize, expected_stderr=expected_stderr_with_optimize) + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True), [], tool], expected_logs=expected_logs, expected_stdout=expected_stdout, expected_stderr=expected_stderr_with_optimize) def test_overrides_are_included_correctly(self): command = RebaselineExpectations() @@ -262,3 +332,151 @@ MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Win', port._filesystem.write_text_file(port.layout_tests_dir() + '/userscripts/another-test.html', '') self.assertEquals(command._tests_to_rebaseline(port), {'userscripts/another-test.html': set(['txt'])}) self.assertEquals(port._filesystem.read_text_file(expectations_path), expectations_contents) + + def test_rebaseline(self): + old_exact_matches = builders._exact_matches + try: + builders._exact_matches = { + "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, + } + + command = Rebaseline() + tool = MockTool() + command.bind_to_tool(tool) + + for port_name in tool.port_factory.all_port_names(): + port = tool.port_factory.get(port_name) + for path in port.expectations_files(): + tool.filesystem.write_text_file(path, '') + + tool.executive = MockExecutive(should_log=True) + + def mock_builders_to_pull_from(): + return [MockBuilder('MOCK builder')] + + def mock_tests_to_update(build): + return ['mock/path/to/test.html'] + + command._builders_to_pull_from = mock_builders_to_pull_from + command._tests_to_update = mock_tests_to_update + + expected_stdout = """rebaseline-json: {'mock/path/to/test.html': {'MOCK builder': ['txt']}} +""" + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'mock/path/to/test.html'], cwd=/mock-checkout +""" + + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True, builders=None, suffixes=["txt"], verbose=True), [], tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) + + finally: + builders._exact_matches = old_exact_matches + + def test_rebaseline_command_line_flags(self): + old_exact_matches = builders._exact_matches + try: + builders._exact_matches = { + "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, + } + + command = Rebaseline() + tool = MockTool() + command.bind_to_tool(tool) + + for port_name in tool.port_factory.all_port_names(): + port = tool.port_factory.get(port_name) + for path in port.expectations_files(): + tool.filesystem.write_text_file(path, '') + + tool.executive = MockExecutive(should_log=True) + + expected_stdout = """rebaseline-json: {'mock/path/to/test.html': {'MOCK builder': ['txt']}} +""" + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'mock/path/to/test.html'], cwd=/mock-checkout +""" + + builder = "MOCK builder" + test = "mock/path/to/test.html" + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True, builders=[builder], suffixes=["txt"], verbose=True), [test], tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) + + finally: + builders._exact_matches = old_exact_matches + + def test_rebaseline_multiple_builders(self): + old_exact_matches = builders._exact_matches + try: + builders._exact_matches = { + "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, + "MOCK builder2": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier2"])}, + } + + command = Rebaseline() + tool = MockTool() + command.bind_to_tool(tool) + + for port_name in tool.port_factory.all_port_names(): + port = tool.port_factory.get(port_name) + for path in port.expectations_files(): + tool.filesystem.write_text_file(path, '') + + tool.executive = MockExecutive(should_log=True) + + def mock_builders_to_pull_from(): + return [MockBuilder('MOCK builder'), MockBuilder('MOCK builder2')] + + def mock_tests_to_update(build): + return ['mock/path/to/test.html'] + + command._builders_to_pull_from = mock_builders_to_pull_from + command._tests_to_update = mock_tests_to_update + + expected_stdout = """rebaseline-json: {'mock/path/to/test.html': {'MOCK builder2': ['txt'], 'MOCK builder': ['txt']}} +""" + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK builder2', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'mock/path/to/test.html'], cwd=/mock-checkout +""" + + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True, builders=None, suffixes=["txt"], verbose=True), [], tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) + + finally: + builders._exact_matches = old_exact_matches + + def test_rebaseline_multiple_builders_and_tests_command_line(self): + old_exact_matches = builders._exact_matches + try: + builders._exact_matches = { + "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, + "MOCK builder2": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier2"])}, + "MOCK builder3": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier2"])}, + } + + command = Rebaseline() + tool = MockTool() + command.bind_to_tool(tool) + + for port_name in tool.port_factory.all_port_names(): + port = tool.port_factory.get(port_name) + for path in port.expectations_files(): + tool.filesystem.write_text_file(path, '') + + tool.executive = MockExecutive(should_log=True) + + expected_stdout = """rebaseline-json: {'mock/path/to/test.html': {'MOCK builder2': ['txt', 'png', 'wav'], 'MOCK builder': ['txt', 'png', 'wav'], 'MOCK builder3': ['txt', 'png', 'wav']}, 'mock/path/to/test2.html': {'MOCK builder2': ['txt', 'png', 'wav'], 'MOCK builder': ['txt', 'png', 'wav'], 'MOCK builder3': ['txt', 'png', 'wav']}} +""" + + expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png,wav', '--builder', 'MOCK builder2', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png,wav', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png,wav', '--builder', 'MOCK builder2', '--test', 'mock/path/to/test2.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png,wav', '--builder', 'MOCK builder', '--test', 'mock/path/to/test2.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'wav,txt,png', 'mock/path/to/test.html'], cwd=/mock-checkout +MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'wav,txt,png', 'mock/path/to/test2.html'], cwd=/mock-checkout +""" + + OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True, builders=["MOCK builder,MOCK builder2", "MOCK builder3"], suffixes=["txt", "png,wav"], verbose=True), ["mock/path/to/test.html", "mock/path/to/test2.html"], tool], expected_stdout=expected_stdout, expected_stderr=expected_stderr) + + finally: + builders._exact_matches = old_exact_matches diff --git a/Tools/Scripts/webkitpy/tool/mocktool.py b/Tools/Scripts/webkitpy/tool/mocktool.py index 21ee91fc9..b8f0976bc 100644 --- a/Tools/Scripts/webkitpy/tool/mocktool.py +++ b/Tools/Scripts/webkitpy/tool/mocktool.py @@ -29,6 +29,7 @@ import threading from webkitpy.common.host_mock import MockHost +from webkitpy.common.net.buildbot.buildbot_mock import MockBuildBot from webkitpy.common.net.statusserver_mock import MockStatusServer from webkitpy.common.net.irc.irc_mock import MockIRC @@ -82,3 +83,6 @@ class MockTool(MockHost): def irc(self): return self._irc + + def buildbot_for_builder_name(self, name): + return MockBuildBot() diff --git a/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py b/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py index bfe003fe9..947bf1d88 100644 --- a/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py +++ b/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py @@ -140,79 +140,7 @@ class GardeningHTTPRequestHandler(ReflectionHandler): self._expectations_updater().update_expectations(self._read_entity_body_as_json()) self._serve_text('success') - def rebaseline(self): - builder = self.query['builder'][0] - command = [ 'rebaseline-test' ] - - if 'suffixes' in self.query: - command.append('--suffixes') - command.append(self.query['suffixes'][0]) - - command.append(builder) - command.append(self.query['test'][0]) - - command.extend(builders.fallback_port_names_for_new_port(builder)) - self._run_webkit_patch(command) - self._serve_text('success') - - def _builders_to_fetch_from(self, builders): - # This routine returns the subset of builders that will cover all of the baseline search paths - # used in the input list. In particular, if the input list contains both Release and Debug - # versions of a configuration, we *only* return the Release version (since we don't save - # debug versions of baselines). - release_builders = set() - debug_builders = set() - builders_to_fallback_paths = {} - for builder in builders: - port = self.server.tool.port_factory.get_from_builder_name(builder) - if port.test_configuration().build_type == 'Release': - release_builders.add(builder) - else: - debug_builders.add(builder) - for builder in list(release_builders) + list(debug_builders): - port = self.server.tool.port_factory.get_from_builder_name(builder) - fallback_path = port.baseline_search_path() - if fallback_path not in builders_to_fallback_paths.values(): - builders_to_fallback_paths[builder] = fallback_path - return builders_to_fallback_paths.keys() - - def _rebaseline_commands(self, test_list): - path_to_webkit_patch = self.server.tool.path() - cwd = self.server.tool.scm().checkout_root - commands = [] - for test in test_list: - for builder in self._builders_to_fetch_from(test_list[test]): - suffixes = ','.join(test_list[test][builder]) - cmd_line = [path_to_webkit_patch, 'rebaseline-test', '--print-scm-changes', '--suffixes', suffixes, builder, test] - commands.append(tuple([cmd_line, cwd])) - return commands - - def _files_to_add(self, command_results): - files_to_add = set() - for output in [result[1] for result in command_results]: - try: - files_to_add.update(json.loads(output)['add']) - except ValueError, e: - _log.warning('"%s" is not a JSON object, ignoring' % output) - - return list(files_to_add) - - def _optimize_baselines(self, test_list): - # We don't run this in parallel because modifying the SCM in parallel is unreliable. - for test in test_list: - all_suffixes = set() - for builder in self._builders_to_fetch_from(test_list[test]): - all_suffixes.update(test_list[test][builder]) - self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test]) - def rebaselineall(self): - test_list = self._read_entity_body_as_json() - - commands = self._rebaseline_commands(test_list) - command_results = self.server.tool.executive.run_in_parallel(commands) - - files_to_add = self._files_to_add(command_results) - self.server.tool.scm().add_list(list(files_to_add)) - - self._optimize_baselines(test_list) + command = ['rebaseline-json'] + self.server.tool.executive.run_command([self.server.tool.path()] + command, input=self.read_entity_body(), cwd=self.server.tool.scm().checkout_root) self._serve_text('success') diff --git a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py index 4cc772ca2..166d191ac 100644 --- a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py +++ b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py @@ -69,7 +69,7 @@ class TestGardeningHTTPRequestHandler(GardeningHTTPRequestHandler): def _expectations_updater(self): return GardeningExpectationsUpdater(self.server.tool, TestPortFactory.create()) - def _read_entity_body(self): + def read_entity_body(self): return self.body if self.body else '' def _serve_text(self, text): @@ -183,32 +183,26 @@ class GardeningServerTest(unittest.TestCase): self._post_to_path("/rollout?revision=2314&reason=MOCK+rollout+reason", expected_stderr=expected_stderr, expected_stdout=expected_stdout) def test_rebaselineall(self): - builders._exact_matches = { - "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, - "MOCK builder (Debug)": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier", "debug"])}, - } - expected_stderr = "MOCK run_command: ['echo', 'rebaseline-test', '--print-scm-changes', '--suffixes', u'%s', u'%s', u'user-scripts/another-test.html'], cwd=/mock-checkout\nMOCK run_command: ['echo', 'optimize-baselines', '--suffixes', u'%s', u'user-scripts/another-test.html'], cwd=/mock-checkout\n" + expected_stderr = "MOCK run_command: ['echo', 'rebaseline-json'], cwd=/mock-checkout, input={\"user-scripts/another-test.html\":{\"%s\": [%s]}}\n" expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n" server = MockServer() self.output = ['{"add": [], "delete": []}', ''] - def run_command(args, cwd=None, **kwargs): - print >> sys.stderr, "MOCK run_command: %s, cwd=%s" % (args, cwd) + def run_command(args, cwd=None, input=None, **kwargs): + print >> sys.stderr, "MOCK run_command: %s, cwd=%s, input=%s" % (args, cwd, input) return self.output.pop(0) server.tool.executive.run_command = run_command - self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}', expected_stderr=expected_stderr % ('txt,png', 'MOCK builder', 'txt,png'), expected_stdout=expected_stdout, server=server) + self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}', expected_stderr=expected_stderr % ('MOCK builder', '"txt","png"'), expected_stdout=expected_stdout, server=server) - self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"]}}', expected_stderr=expected_stderr % ('txt,png', 'MOCK builder (Debug)', 'txt,png'), expected_stdout=expected_stdout) - - self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"], "MOCK builder": ["txt"]}}', expected_stderr=expected_stderr % ('txt', 'MOCK builder', 'txt'), expected_stdout=expected_stdout) + self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"]}}', expected_stderr=expected_stderr % ('MOCK builder (Debug)', '"txt","png"'), expected_stdout=expected_stdout) def test_rebaseline_new_port(self): builders._exact_matches = {"MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"]), "move_overwritten_baselines_to": ["mock-port-fallback", "mock-port-fallback2"]}} - expected_stderr = "MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt,png', 'MOCK builder', 'user-scripts/another-test.html', 'mock-port-fallback', 'mock-port-fallback2'], cwd=/mock-checkout\n" + expected_stderr = 'MOCK run_command: [\'echo\', \'rebaseline-json\'], cwd=/mock-checkout, input={"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}\n' expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n" - self._post_to_path("/rebaseline?builder=MOCK+builder&test=user-scripts/another-test.html&suffixes=txt,png", expected_stderr=expected_stderr, expected_stdout=expected_stdout) + self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}', expected_stderr=expected_stderr, expected_stdout=expected_stdout) def test_updateexpectations(self): expected_stderr = "" diff --git a/Tools/Scripts/webkitpy/tool/servers/reflectionhandler.py b/Tools/Scripts/webkitpy/tool/servers/reflectionhandler.py index 6a3f207be..24bb2771a 100644 --- a/Tools/Scripts/webkitpy/tool/servers/reflectionhandler.py +++ b/Tools/Scripts/webkitpy/tool/servers/reflectionhandler.py @@ -59,12 +59,12 @@ class ReflectionHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_POST(self): self._handle_request() - def _read_entity_body(self): + def read_entity_body(self): length = int(self.headers.getheader('content-length')) return self.rfile.read(length) def _read_entity_body_as_json(self): - return json.loads(self._read_entity_body()) + return json.loads(self.read_entity_body()) def _handle_request(self): if "?" in self.path: diff --git a/Tools/Scripts/webkitpy/tool/steps/__init__.py b/Tools/Scripts/webkitpy/tool/steps/__init__.py index aca9706f5..56429e8fe 100644 --- a/Tools/Scripts/webkitpy/tool/steps/__init__.py +++ b/Tools/Scripts/webkitpy/tool/steps/__init__.py @@ -27,6 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # FIXME: Is this the right way to do this? +from webkitpy.tool.steps.addsvnmimetypeforpng import AddSvnMimetypeForPng from webkitpy.tool.steps.applypatch import ApplyPatch from webkitpy.tool.steps.applypatchwithlocalcommit import ApplyPatchWithLocalCommit from webkitpy.tool.steps.applywatchlist import ApplyWatchList diff --git a/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng.py b/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng.py new file mode 100644 index 000000000..73bec15db --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng.py @@ -0,0 +1,77 @@ +# Copyright (C) 2012 Balazs Ankes (bank@inf.u-szeged.hu) University of Szeged +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from webkitpy.tool.steps.abstractstep import AbstractStep +from webkitpy.common import checksvnconfigfile +from webkitpy.common.system.deprecated_logging import log +from webkitpy.common.checkout.scm.detection import SCMDetector +from webkitpy.common.system.systemhost import SystemHost + + +class AddSvnMimetypeForPng(AbstractStep): + def __init__(self, tool, options, host=None, scm=None): + self._tool = tool + self._options = options + self._host = host or SystemHost() + self._fs = self._host.filesystem + self._detector = scm or SCMDetector(self._fs, self._host.executive).detect_scm_system(self._fs.getcwd()) + + def run(self, state): + png_files = self._check_pngs(self._changed_files(state)) + + if png_files: + detection = self._detector.display_name() + + if detection == "git": + (file_missing, autoprop_missing, png_missing) = checksvnconfigfile.check(self._host, self._fs) + config_file_path = checksvnconfigfile.config_file_path(self._host, self._fs) + + if file_missing: + log("There is no SVN config file. The svn:mime-type of pngs won't set.") + if not self._tool.user.confirm("Are you sure you want to continue?", default="n"): + self._exit(1) + elif autoprop_missing and png_missing: + log(checksvnconfigfile.errorstr_autoprop(config_file_path) + checksvnconfigfile.errorstr_png(config_file_path)) + if not self._tool.user.confirm("Do you want to continue?", default="n"): + self._exit(1) + elif autoprop_missing: + log(checksvnconfigfile.errorstr_autoprop(config_file_path)) + if not self._tool.user.confirm("Do you want to continue?", default="n"): + self._exit(1) + elif png_missing: + log(checksvnconfigfile.errorstr_png(config_file_path)) + if not self._tool.user.confirm("Do you want to continue?", default="n"): + self._exit(1) + + elif detection == "svn": + for filename in png_files: + if self._detector.exists(filename) and self._detector.propget('svn:mime-type', filename) != 'image/png': + print "Adding image/png mime-type to %s" % filename + self._detector.propset('svn:mime-type', 'image/png', filename) + + def _check_pngs(self, changed_files): + png_files = [] + for filename in changed_files: + if filename.endswith('.png'): + png_files.append(filename) + return png_files diff --git a/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng_unittest.py b/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng_unittest.py new file mode 100644 index 000000000..221c6bc94 --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/steps/addsvnmimetypeforpng_unittest.py @@ -0,0 +1,58 @@ +# Copyright (C) 2012 Balazs Ankes (bank@inf.u-szeged.hu) University of Szeged +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.tool.steps.addsvnmimetypeforpng import AddSvnMimetypeForPng +from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.tool.mocktool import MockOptions, MockTool +from webkitpy.common.system.systemhost_mock import MockSystemHost +from webkitpy.common.system.outputcapture import OutputCapture + + +class MockSCMDetector(object): + + def __init__(self, scm): + self._scm = scm + + def display_name(self): + return self._scm + + +class AddSvnMimetypeForPngTest(unittest.TestCase): + def test_run(self): + capture = OutputCapture() + options = MockOptions(git_commit='MOCK git commit') + + files = {'/Users/mock/.subversion/config': 'enable-auto-props = yes\n*.png = svn:mime-type=image/png'} + fs = MockFileSystem(files) + scm = MockSCMDetector('git') + + step = AddSvnMimetypeForPng(MockTool(), options, MockSystemHost(os_name='linux', filesystem=fs), scm) + state = { + "changed_files": ["test.png", "test.txt"], + } + try: + capture.assert_outputs(self, step.run, [state]) + except SystemExit, e: + self.assertEquals(e.code, 1) diff --git a/Tools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py b/Tools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py index ffed201d2..847dc2f93 100644 --- a/Tools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py +++ b/Tools/Scripts/webkitpy/tool/steps/preparechangelog_unittest.py @@ -52,6 +52,6 @@ class PrepareChangeLogTest(changelog_unittest.ChangeLogTest): capture.assert_outputs(self, step.run, [state]) actual_contents = self._read_file_contents(changelog_path, "utf-8") expected_message = "Example title\n http://example.com/1234" - expected_contents = changelog_contents.replace("Need a short description and bug URL (OOPS!)", expected_message) + expected_contents = changelog_contents.replace("Need a short description (OOPS!).\n Need the bug URL (OOPS!).", expected_message) os.remove(changelog_path) self.assertEquals(actual_contents.splitlines(), expected_contents.splitlines()) |