diff options
| author | Simon Hausmann <simon.hausmann@digia.com> | 2012-10-15 16:08:57 +0200 |
|---|---|---|
| committer | Simon Hausmann <simon.hausmann@digia.com> | 2012-10-15 16:08:57 +0200 |
| commit | 5466563f4b5b6b86523e3f89bb7f77e5b5270c78 (patch) | |
| tree | 8caccf7cd03a15207cde3ba282c88bf132482a91 /Tools/Scripts/webkitpy | |
| parent | 33b26980cb24288b5a9f2590ccf32a949281bb79 (diff) | |
| download | qtwebkit-5466563f4b5b6b86523e3f89bb7f77e5b5270c78.tar.gz | |
Imported WebKit commit 0dc6cd75e1d4836eaffbb520be96fac4847cc9d2 (http://svn.webkit.org/repository/webkit/trunk@131300)
WebKit update which introduces the QtWebKitWidgets module that contains the WK1
widgets based API. (In fact it renames QtWebKit to QtWebKitWidgets while we're
working on completing the entire split as part of
https://bugs.webkit.org/show_bug.cgi?id=99314
Diffstat (limited to 'Tools/Scripts/webkitpy')
67 files changed, 3550 insertions, 1170 deletions
diff --git a/Tools/Scripts/webkitpy/common/config/committers.py b/Tools/Scripts/webkitpy/common/config/committers.py index e98b25936..1c231d37a 100644 --- a/Tools/Scripts/webkitpy/common/config/committers.py +++ b/Tools/Scripts/webkitpy/common/config/committers.py @@ -108,18 +108,17 @@ watchers_who_are_not_contributors = [ contributors_who_are_not_committers = [ Contributor("Aharon Lanin", "aharon@google.com"), - Contributor("Alan Stearns", "stearns@adobe.com", 'astearns'), + Contributor("Alan Stearns", "stearns@adobe.com", "astearns"), Contributor("Alexey Marinichev", ["amarinichev@chromium.org", "amarinichev@google.com"], "amarinichev"), Contributor("Andras Piroska", "pandras@inf.u-szeged.hu", "andris88"), Contributor("Andrei Bucur", "abucur@adobe.com", "abucur"), Contributor("Anne van Kesteren", "annevankesteren+webkit@gmail.com", "annevk"), Contributor("Annie Sullivan", "sullivan@chromium.org", "annie"), Contributor("Aryeh Gregor", "ayg@aryeh.name", "AryehGregor"), - Contributor("Balazs Ankes", "bank@inf.u-szeged.hu", 'abalazs'), + Contributor("Balazs Ankes", "bank@inf.u-szeged.hu", "abalazs"), Contributor("Brian Salomon", "bsalomon@google.com"), Contributor("Commit Queue", "commit-queue@webkit.org"), Contributor("Daniel Sievers", "sievers@chromium.org"), - Contributor("David Barr", "davidbarr@chromium.org", "barrbrain"), Contributor("David Dorwin", "ddorwin@chromium.org", "ddorwin"), Contributor("David Reveman", "reveman@chromium.org", "reveman"), Contributor("Dongsung Huang", "luxtella@company100.net", "Huang"), @@ -136,8 +135,8 @@ contributors_who_are_not_committers = [ Contributor("Gregg Tavares", ["gman@google.com", "gman@chromium.org"], "gman"), Contributor("Hao Zheng", "zhenghao@chromium.org"), Contributor("Ian Hickson", "ian@hixie.ch", "hixie"), - Contributor("Janos Badics", "jbadics@inf.u-szeged.hu", 'dicska'), - Contributor("Jonathan Backer", "backer@chromium.org", 'backer'), + Contributor("Janos Badics", "jbadics@inf.u-szeged.hu", "dicska"), + Contributor("Jonathan Backer", "backer@chromium.org", "backer"), Contributor("Jeff Timanus", ["twiz@chromium.org", "twiz@google.com"], "twiz"), Contributor("Jing Zhao", "jingzhao@chromium.org"), Contributor("John Bates", ["jbates@google.com", "jbates@chromium.org"], "jbates"), @@ -151,7 +150,7 @@ contributors_who_are_not_committers = [ Contributor("Oliver Varga", ["voliver@inf.u-szeged.hu", "Varga.Oliver@stud.u-szeged.hu"], "TwistO"), Contributor("Peter Gal", "galpeter@inf.u-szeged.hu", "elecro"), Contributor("Peter Linss", "peter.linss@hp.com", "plinss"), - Contributor("Pravin D", "pravind.2k4@gmail.com", 'pravind'), + Contributor("Pravin D", "pravind.2k4@gmail.com", "pravind"), Contributor("Radar WebKit Bug Importer", "webkit-bug-importer@group.apple.com"), Contributor("Raul Hudea", "rhudea@adobe.com", "rhudea"), Contributor("Roland Takacs", "rtakacs@inf.u-szeged.hu", "rtakacs"), @@ -179,7 +178,7 @@ contributors_who_are_not_committers = [ committers_unable_to_review = [ Committer("Aaron Boodman", "aa@chromium.org", "aboodman"), Committer("Adam Bergkvist", "adam.bergkvist@ericsson.com", "adambe"), - Committer("Adam Kallai", "kadam@inf.u-szeged.hu", 'kadam'), + Committer("Adam Kallai", "kadam@inf.u-szeged.hu", "kadam"), Committer("Adam Klein", "adamk@chromium.org", "aklein"), Committer("Adam Langley", "agl@chromium.org", "agl"), Committer("Ademar de Souza Reis Jr", ["ademar.reis@gmail.com", "ademar@webkit.org"], "ademar"), @@ -216,7 +215,6 @@ committers_unable_to_review = [ Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"], "otte"), Committer("Bill Budge", ["bbudge@chromium.org", "bbudge@gmail.com"], "bbudge"), Committer("Brett Wilson", "brettw@chromium.org", "brettx"), - Committer("Caio Marcelo de Oliveira Filho", ["cmarcelo@webkit.org", "caio.oliveira@openbossa.org"], "cmarcelo"), Committer("Cameron McCormack", ["cam@mcc.id.au", "cam@webkit.org"], "heycam"), Committer("Carol Szabo", ["carol@webkit.org", "carol.szabo@nokia.com"], "cszabo1"), Committer("Cary Clark", ["caryclark@google.com", "caryclark@chromium.org"], "caryclark"), @@ -233,9 +231,11 @@ committers_unable_to_review = [ Committer("Daniel Cheng", "dcheng@chromium.org", "dcheng"), Committer("Dave Barton", "dbarton@mathscribe.com", "dbarton"), Committer("Dave Tharp", "dtharp@codeaurora.org", "dtharp"), + Committer("David Michael Barr", ["davidbarr@chromium.org", "davidbarr@google.com", "b@rr-dav.id.au"], "barrbrain"), Committer("David Grogan", ["dgrogan@chromium.org", "dgrogan@google.com"], "dgrogan"), Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"], "catfishman"), Committer("Diego Gonzalez", ["diegohcg@webkit.org", "diego.gonzalez@openbossa.org"], "diegohcg"), + Committer("Dinu Jacob", "dinu.s.jacob@intel.com", "dsjacob"), Committer("Dmitry Lomov", ["dslomov@google.com", "dslomov@chromium.org"], "dslomov"), Committer("Dominic Cooney", ["dominicc@chromium.org", "dominicc@google.com"], "dominicc"), Committer("Dominic Mazzoni", ["dmazzoni@google.com", "dmazzoni@chromium.org"], "dmazzoni"), @@ -243,7 +243,6 @@ committers_unable_to_review = [ Committer("Drew Wilson", "atwilson@chromium.org", "atwilson"), Committer("Eli Fidler", ["eli@staikos.net", "efidler@rim.com"], "efidler"), Committer("Elliot Poger", "epoger@chromium.org", "epoger"), - Committer("Emil A Eklund", "eae@chromium.org", "eae"), Committer("Erik Arvidsson", "arv@chromium.org", "arv"), Committer("Eric Roman", "eroman@chromium.org", "eroman"), Committer("Eric Uhrhane", "ericu@chromium.org", "ericu"), @@ -299,6 +298,7 @@ committers_unable_to_review = [ Committer("Joshua Bell", ["jsbell@chromium.org", "jsbell@google.com"], "jsbell"), Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"], "jparent"), Committer("Jungshik Shin", "jshin@chromium.org"), + Committer("Justin Novosad", ["junov@google.com", "junov@chromium.org"], "junov"), Committer("Justin Schuh", "jschuh@chromium.org", "jschuh"), Committer("Kaustubh Atrawalkar", ["kaustubh@motorola.com"], "silverroots"), Committer("Keishi Hattori", "keishi@webkit.org", "keishi"), @@ -378,6 +378,7 @@ committers_unable_to_review = [ Committer("Siddharth Mathur", "siddharth.mathur@nokia.com", "simathur"), Committer("Stephen Chenney", "schenney@chromium.org", "schenney"), Committer("Steve Lacey", "sjl@chromium.org", "stevela"), + Committer("Taiju Tsuiki", "tzik@chromium.org", "tzik"), Committer("Takashi Toyoshima", "toyoshim@chromium.org", "toyoshim"), Committer("Thomas Sepez", "tsepez@chromium.org", "tsepez"), Committer("Tom Hudson", ["tomhudson@google.com", "tomhudson@chromium.org"], "tomhudson"), @@ -441,6 +442,7 @@ reviewers_list = [ Reviewer("Brady Eidson", "beidson@apple.com", "bradee-oh"), Reviewer("Brent Fulgham", "bfulgham@webkit.org", "bfulgham"), Reviewer("Brian Weinstein", "bweinstein@apple.com", "bweinstein"), + Reviewer("Caio Marcelo de Oliveira Filho", ["cmarcelo@webkit.org", "caio.oliveira@openbossa.org"], "cmarcelo"), Reviewer("Cameron Zwarich", ["zwarich@apple.com", "cwzwarich@apple.com", "cwzwarich@webkit.org"]), Reviewer("Carlos Garcia Campos", ["cgarcia@igalia.com", "carlosgc@gnome.org", "carlosgc@webkit.org"], "KaL"), Reviewer("Chang Shu", ["cshu@webkit.org", "c.shu@sisa.samsung.com"], "cshu"), @@ -465,6 +467,7 @@ reviewers_list = [ Reviewer("Dmitry Titov", "dimich@chromium.org", "dimich"), Reviewer("Don Melton", "gramps@apple.com", "gramps"), Reviewer("Dumitru Daniliuc", "dumi@chromium.org", "dumi"), + Reviewer("Emil A Eklund", "eae@chromium.org", "eae"), Reviewer("Enrica Casucci", "enrica@apple.com", "enrica"), Reviewer("Eric Carlson", "eric.carlson@apple.com", "eric_carlson"), Reviewer("Eric Seidel", "eric@webkit.org", "eseidel"), @@ -497,7 +500,7 @@ reviewers_list = [ Reviewer("Kevin McCullough", "kmccullough@apple.com", "maculloch"), Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"], "kollivier"), Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org", "lars.knoll@nokia.com"], "lars"), - Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com", "lgombos"), + Reviewer("Laszlo Gombos", ["laszlo.gombos@webkit.org", "l.gombos@samsung.com", "laszlo.1.gombos@nokia.com"], "lgombos"), Reviewer("Levi Weintraub", ["leviw@chromium.org", "leviw@google.com", "lweintraub@apple.com"], "leviw"), Reviewer("Luiz Agostini", ["luiz@webkit.org", "luiz.agostini@openbossa.org"], "lca"), Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), diff --git a/Tools/Scripts/webkitpy/common/config/watchlist b/Tools/Scripts/webkitpy/common/config/watchlist index 2ab16b0c2..9b42b7427 100755 --- a/Tools/Scripts/webkitpy/common/config/watchlist +++ b/Tools/Scripts/webkitpy/common/config/watchlist @@ -56,6 +56,15 @@ "webkitpy": { "filename": r"Tools/Scripts/webkitpy/", }, + "webkitperl": { + "filename": r"Tools/Scripts/webkitperl/" + r"|Tools/Scripts/webkitdirs.pm" + r"|Tools/Scripts/VCSUtils.pm" + r"|Tools/Scripts/test-webkitperl", + }, + "SVNScripts": { + "filename": r"Tools/Scripts/svn-.*", + }, "TestFailures": { "filename": r"Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/", }, @@ -64,6 +73,9 @@ "less": r"[Ss]ecurityOrigin(?!\.(h|cpp))", "filename": r"XSS|[Ss]ecurity", }, + "XSS": { + "filename": r".*XSS", + }, "SkiaGraphics": { "filename": r"Source/WebCore/platform/graphics/skia/" r"|Source/WebCore/platform/graphics/filters/skia/", @@ -236,6 +248,18 @@ "filename": r"Source/WebCore/svg" r"|Source/WebCore/rendering/svg", }, + "WebInspectorProtocol": { + "filename": r"Source/WebCore/inspector/Inspector.json", + }, + "WebSocket": { + "filename": r"Source/WebCore/Modules/websockets" + r"|Source/WebCore/platform/network/(|.+/)SocketStream.*", + }, + "MediaStream": { + "filename": r"Source/WebCore/Modules/mediastream" + r"|Source/WebCore/platform/mediastream" + r"|LayoutTests/fast/mediastream", + }, }, "CC_RULES": { # Note: All email addresses listed must be registered with bugzilla. @@ -243,7 +267,7 @@ # two different accounts as far as bugzilla is concerned. "AppleMacPublicApi": [ "timothy@apple.com" ], "Battery": [ "gyuyoung.kim@samsung.com" ], - "BlackBerry": [ "mifenton@rim.com", "rwlbuis@gmail.com" ], + "BlackBerry": [ "mifenton@rim.com", "rwlbuis@gmail.com", "tonikitoo@webkit.org" ], "CMake": [ "rakuco@webkit.org", "gyuyoung.kim@samsung.com" ], "CSS": [ "alexis@webkit.org", "macpherson@chromium.org", "cmarcelo@webkit.org" ], "ChromiumGraphics": [ "jamesr@chromium.org", "cc-bugs@chromium.org" ], @@ -261,29 +285,35 @@ "Loader": [ "japhet@chromium.org" ], "MathML": [ "dbarton@mathscribe.com" ], "Media": [ "feature-media-reviews@chromium.org", "eric.carlson@apple.com" ], + "MediaStream": [ "tommyw@google.com" ], "NetworkInfo": [ "gyuyoung.kim@samsung.com" ], "OpenGL" : [ "noam.rosenthal@nokia.com", "dino@apple.com" ], - "SkiaGraphics": [ "senorblanco@chromium.org" ], "QtBuildSystem" : [ "vestbo@webkit.org", "abecsi@webkit.org" ], "QtGraphics" : [ "noam.rosenthal@nokia.com" ], - "QtWebKit2PlatformSpecific": [ "alexis@webkit.org", "zoltan@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], - "QtWebKit2PublicAPI": [ "alexis@webkit.org", "zoltan@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], + "QtWebKit2PlatformSpecific": [ "alexis@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], + "QtWebKit2PublicAPI": [ "alexis@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], "Rendering": [ "eric@webkit.org" ], + "SVG": ["schenney@chromium.org", "pdr@google.com", "fmalita@chromium.org" ], + "SVNScripts": [ "dbates@webkit.org" ], "ScrollingCoordinator": [ "andersca@apple.com", "jamesr@chromium.org", "tonikitoo@webkit.org" ], "SecurityCritical": [ "abarth@webkit.org" ], + "SkiaGraphics": [ "senorblanco@chromium.org" ], "SoupNetwork": [ "rakuco@webkit.org", "gns@gnome.org", "mrobinson@webkit.org", "danw@gnome.org" ], "StyleChecker": [ "levin@chromium.org", ], - "SVG": ["schenney@chromium.org", "pdr@google.com", "fmalita@chromium.org" ], "TestFailures": [ "abarth@webkit.org", "dglazkov@chromium.org" ], "TextureMapper" : [ "noam.rosenthal@nokia.com" ], "ThreadingFiles|ThreadingUsage": [ "levin+threading@chromium.org", ], "V8Bindings|BindingsScripts": [ "abarth@webkit.org", "japhet@chromium.org", "haraken@chromium.org" ], + "WTF": [ "benjamin@webkit.org",], "WatchListScript": [ "levin+watchlist@chromium.org", ], "WebGL": [ "dino@apple.com" ], "WebIDL": [ "abarth@webkit.org", "ojan@chromium.org" ], + "WebInspectorProtocol": [ "timothy@apple.com", ], "WebKitGTKTranslations": [ "gns@gnome.org", "mrobinson@webkit.org" ], + "WebSocket": [ "yutak@chromium.org" ], + "XSS": [ "dbates@webkit.org" ], + "webkitperl": [ "dbates@webkit.org" ], "webkitpy": [ "abarth@webkit.org", "ojan@chromium.org", "dpranke@chromium.org" ], - "WTF": [ "benjamin@webkit.org",], }, "MESSAGE_RULES": { "ChromiumPublicApi": [ "Please wait for approval from abarth@webkit.org, dglazkov@chromium.org, " diff --git a/Tools/Scripts/webkitpy/common/host.py b/Tools/Scripts/webkitpy/common/host.py index 53889657b..7dd5ad024 100644 --- a/Tools/Scripts/webkitpy/common/host.py +++ b/Tools/Scripts/webkitpy/common/host.py @@ -125,7 +125,7 @@ class Host(SystemHost): except OSError, e: _log.debug('Failed to engage git.bat Windows hack.') - def _initialize_scm(self, patch_directories=None): + def initialize_scm(self, patch_directories=None): if sys.platform == "win32": self._engage_awesome_windows_hacks() detector = SCMDetector(self.filesystem, self.executive) diff --git a/Tools/Scripts/webkitpy/common/host_mock.py b/Tools/Scripts/webkitpy/common/host_mock.py index ca4f78eb3..8b508bf8f 100644 --- a/Tools/Scripts/webkitpy/common/host_mock.py +++ b/Tools/Scripts/webkitpy/common/host_mock.py @@ -50,7 +50,7 @@ class MockHost(MockSystemHost): # FIXME: we should never initialize the SCM by default, since the real # object doesn't either. This has caused at least one bug (see bug 89498). if initialize_scm_by_default: - self._initialize_scm() + self.initialize_scm() self.bugs = MockBugzilla() self.buildbot = MockBuildBot() self._chromium_buildbot = MockBuildBot() @@ -61,7 +61,7 @@ class MockHost(MockSystemHost): self._watch_list = MockWatchList() - def _initialize_scm(self, patch_directories=None): + def initialize_scm(self, patch_directories=None): self._scm = MockSCM(filesystem=self.filesystem, executive=self.executive) # Various pieces of code (wrongly) call filesystem.chdir(checkout_root). # Making the checkout_root exist in the mock filesystem makes that chdir not raise. diff --git a/Tools/Scripts/webkitpy/common/system/executive_unittest.py b/Tools/Scripts/webkitpy/common/system/executive_unittest.py index 8a322e480..57c557369 100644 --- a/Tools/Scripts/webkitpy/common/system/executive_unittest.py +++ b/Tools/Scripts/webkitpy/common/system/executive_unittest.py @@ -162,6 +162,10 @@ class ExecutiveTest(unittest.TestCase): # FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790 # We seem to get either 0 or 1 here for some reason. self.assertTrue(process.wait() in (0, 1)) + elif sys.platform == "cygwin": + # FIXME: https://bugs.webkit.org/show_bug.cgi?id=98196 + # cygwin seems to give us either SIGABRT or SIGKILL + self.assertTrue(process.wait() in (-signal.SIGABRT, -signal.SIGKILL)) else: expected_exit_code = -signal.SIGKILL self.assertEqual(process.wait(), expected_exit_code) diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py index a7b49831c..6447c8fb4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py @@ -106,11 +106,9 @@ class LayoutTestFinder(object): tests_to_skip = all_tests - tests_to_skip elif self._options.skipped == 'ignore': tests_to_skip = set() - elif self._options.skipped == 'default': - pass # listed for completeness - - # make sure we're explicitly running any tests passed on the command line. - tests_to_skip -= paths + elif self._options.skipped != 'always': + # make sure we're explicitly running any tests passed on the command line; equivalent to 'default'. + tests_to_skip -= paths # unless of course we don't want to run the HTTP tests :) if not self._options.http: diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py index d06ed7153..c0a70e615 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py @@ -239,8 +239,9 @@ def summarize_results(port_obj, expectations, result_summary, retry_summary, tes try: # We only use the svn revision for using trac links in the results.html file, # Don't do this by default since it takes >100ms. + # FIXME: Do we really need to populate this both here and in the json_results_generator? if use_trac_links_in_results_html(port_obj): - port_obj.host._initialize_scm() + port_obj.host.initialize_scm() results['revision'] = port_obj.host.scm().head_svn_revision() except Exception, e: _log.warn("Failed to determine svn revision for checkout (cwd: %s, webkit_base: %s), leaving 'revision' key blank in full_results.json.\n%s" % (port_obj._filesystem.getcwd(), port_obj.path_from_webkit_base(), e)) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index c7d7c3eb7..73834f0ad 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -281,7 +281,8 @@ class JSONResultsGeneratorBase(object): results_for_builder = results_json[builder_name] - self._insert_generic_metadata(results_for_builder) + if builder_name: + self._insert_generic_metadata(results_for_builder) self._insert_failure_summaries(results_for_builder) @@ -377,15 +378,17 @@ class JSONResultsGeneratorBase(object): return self.__class__.PASS_RESULT - # FIXME: Callers should use scm.py instead. - # FIXME: Identify and fix the run-time errors that were observed on Windows - # chromium buildbot when we had updated this code to use scm.py once before. def _get_svn_revision(self, in_directory): """Returns the svn revision for the given directory. Args: in_directory: The directory where svn is to be run. """ + + # FIXME: We initialize this here in order to engage the stupid windows hacks :). + # We can't reuse an existing scm object because the specific directories may + # be part of other checkouts. + self._port.host.initialize_scm() scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(in_directory) if scm: return scm.svn_revision(in_directory) diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py index 42b518f7f..b48c5b933 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py @@ -155,7 +155,7 @@ class TestExpectationParser(object): else: parsed_specifiers.add(modifier) - if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid: + if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations(): expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers: @@ -869,7 +869,7 @@ class TestExpectations(object): return self._model def get_rebaselining_failures(self): - return self._model.get_test_set(REBASELINE, IMAGE) | self._model.get_test_set(REBASELINE, FAIL) + return self._model.get_test_set(REBASELINE) # FIXME: Change the callsites to use TestExpectationsModel and remove. def get_expectations(self, test): @@ -970,7 +970,7 @@ class TestExpectations(object): expectation.name in except_these_tests and 'rebaseline' in expectation.parsed_modifiers)) - return self.list_to_string(filter(without_rebaseline_modifier, self._expectations)) + return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[]) def _add_expectations(self, expectation_list): for expectation_line in expectation_list: diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py index c5606071d..c3fc02658 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py @@ -179,8 +179,7 @@ class MiscTests(Base): "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True) self.assertFalse(True, "ParseError wasn't raised") except ParseError, e: - warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n" - "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n" + warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n" "expectations:2 Path does not exist. non-existent-test.html") self.assertEqual(str(e), warnings) @@ -372,7 +371,9 @@ class SemanticTests(Base): def test_missing_bugid(self): self.parse_exp('failures/expected/text.html [ Failure ]') - self.assertTrue(self._exp.has_warnings()) + self.assertFalse(self._exp.has_warnings()) + + self._port.warn_if_bug_missing_in_test_expectations = lambda: True self.parse_exp('failures/expected/text.html [ Failure ]') line = self._exp._model.get_expectation_line('failures/expected/text.html') @@ -512,8 +513,22 @@ class RebaseliningTest(Base): 'Bug(z) failures/expected/crash.html [ Crash ]\n', 'Bug(x0) failures/expected/image.html [ Crash ]\n') + # Ensure that we don't modify unrelated lines, even if we could rewrite them. + # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html" + self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n' + 'Bug(Y) failures/expected/image.html [ Skip ]\n' + 'Bug(z) failures/expected/crash.html\n', + '', + ['failures/expected/text.html'], + 'Bug(Y) failures/expected/image.html [ Skip ]\n' + 'Bug(z) failures/expected/crash.html\n', + '') + + def test_get_rebaselining_failures(self): + # Make sure we find a test as needing a rebaseline even if it is not marked as a failure. + self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') + self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) - def test_no_get_rebaselining_failures(self): self.parse_exp(self.get_basic_expectations()) self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py index cd57032e9..ae55c684d 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py @@ -182,6 +182,10 @@ class Port(object): """Return the number of DumpRenderTree instances to use for this port.""" return self._executive.cpu_count() + def default_max_locked_shards(self): + """Return the number of "locked" shards to run in parallel (like the http tests).""" + return 1 + def worker_startup_delay_secs(self): # FIXME: If we start workers up too quickly, DumpRenderTree appears # to thrash on something and time out its first few tests. Until @@ -870,6 +874,7 @@ class Port(object): # Most ports (?): 'WEBKIT_TESTFONTS', + 'WEBKITOUTPUTDIR', ] for variable in variables_to_copy: self._copy_value_from_environ_if_set(clean_env, variable) @@ -997,6 +1002,9 @@ class Port(object): # some ports have Skipped files which are returned as part of test_expectations(). return self._filesystem.exists(self.path_to_test_expectations_file()) + def warn_if_bug_missing_in_test_expectations(self): + return False + def expectations_dict(self): """Returns an OrderedDict of name -> expectations strings. The names are expected to be (but not required to be) paths in the filesystem. @@ -1162,9 +1170,18 @@ class Port(object): return "apache2-httpd.conf" def _path_to_apache_config_file(self): - """Returns the full path to the apache binary. + """Returns the full path to the apache configuration file. + + If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its + contents will be used instead. This is needed only by ports that use the apache_http_server module.""" + config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH') + if config_file_from_env: + if not self._filesystem.exists(config_file_from_env): + raise IOError('%s was not found on the system' % config_file_from_env) + return config_file_from_env + config_file_name = self._apache_config_file_name_for_platform(sys.platform) return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name) @@ -1223,11 +1240,15 @@ class Port(object): This is needed only by ports that use the http_server.py module.""" raise NotImplementedError('Port._path_to_lighttpd_php') + @memoized def _path_to_wdiff(self): """Returns the full path to the wdiff binary, or None if it is not available. This is likely used only by wdiff_text()""" - return 'wdiff' + for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): + if self._filesystem.exists(path): + return path + return None def _webkit_baseline_path(self, platform): """Return the full path to the top of the baseline tree for a @@ -1275,7 +1296,7 @@ class Port(object): base_tests = self._real_tests([suite.base]) suite.tests = {} for test in base_tests: - suite.tests[test.replace(suite.base, suite.name)] = test + suite.tests[test.replace(suite.base, suite.name, 1)] = test return suites def _virtual_tests(self, paths, suites): @@ -1292,7 +1313,7 @@ class Port(object): def lookup_virtual_test_base(self, test_name): for suite in self.populated_virtual_test_suites(): if test_name.startswith(suite.name): - return test_name.replace(suite.name, suite.base) + return test_name.replace(suite.name, suite.base, 1) return None def lookup_virtual_test_args(self, test_name): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py index f6a6eb01f..e9b2f060d 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -437,11 +437,15 @@ class PortTest(unittest.TestCase): tests = port.tests(['passes']) self.assertTrue('passes/text.html' in tests) + self.assertTrue('passes/passes/test-virtual-passes.html' in tests) self.assertFalse('virtual/passes/text.html' in tests) tests = port.tests(['virtual/passes']) self.assertFalse('passes/text.html' in tests) - self.assertTrue('virtual/passes/text.html' in tests) + self.assertTrue('virtual/passes/test-virtual-passes.html' in tests) + self.assertTrue('virtual/passes/passes/test-virtual-passes.html' in tests) + self.assertFalse('virtual/passes/test-virtual-virtual/passes.html' in tests) + self.assertFalse('virtual/passes/virtual/passes/test-virtual-passes.html' in tests) def test_build_path(self): port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'})) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/builders.py b/Tools/Scripts/webkitpy/layout_tests/port/builders.py index 3d03fc7af..605b8cccc 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/builders.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/builders.py @@ -37,16 +37,16 @@ from webkitpy.common.memoized import memoized # * specifiers -- a set of specifiers, representing configurations covered by this builder. _exact_matches = { # These builders are on build.chromium.org. - "Webkit Win": {"port_name": "chromium-win-xp", "specifiers": set(["xp", "release"])}, - "Webkit Win7": {"port_name": "chromium-win-win7", "specifiers": set(["win7"])}, - "Webkit Win (dbg)(1)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, - "Webkit Win (dbg)(2)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, - "Webkit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])}, - "Webkit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])}, - "Webkit Linux (dbg)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])}, - "Webkit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])}, - "Webkit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])}, - "Webkit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, + "WebKit XP": {"port_name": "chromium-win-xp", "specifiers": set(["xp", "release"])}, + "WebKit Win7": {"port_name": "chromium-win-win7", "specifiers": set(["win7"])}, + "WebKit Win7 (dbg)(1)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, + "WebKit Win7 (dbg)(2)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, + "WebKit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])}, + "WebKit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])}, + "WebKit Linux (dbg)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])}, + "WebKit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])}, + "WebKit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])}, + "WebKit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, # These builders are on build.webkit.org. "Apple MountainLion Release WK1 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion"]), "rebaseline_override_dir": "mac"}, @@ -92,6 +92,8 @@ _ports_without_builders = [ "qt-mac", "qt-win", "qt-wk2", + # FIXME: Move to _extact_matches. + "chromium-android", ] diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index 44c98a383..c310eb15b 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -53,6 +53,7 @@ class ChromiumPort(Port): ALL_SYSTEMS = ( ('snowleopard', 'x86'), ('lion', 'x86'), + ('mountainlion', 'x86'), ('xp', 'x86'), ('win7', 'x86'), ('lucid', 'x86'), @@ -62,13 +63,13 @@ class ChromiumPort(Port): ('icecreamsandwich', 'x86')) ALL_BASELINE_VARIANTS = [ - 'chromium-mac-lion', 'chromium-mac-snowleopard', 'chromium-mac-leopard', + 'chromium-mac-mountainlion', 'chromium-mac-lion', 'chromium-mac-snowleopard', 'chromium-win-win7', 'chromium-win-xp', 'chromium-linux-x86_64', 'chromium-linux-x86', ] CONFIGURATION_SPECIFIER_MACROS = { - 'mac': ['snowleopard', 'lion'], + 'mac': ['snowleopard', 'lion', 'mountainlion'], 'win': ['xp', 'win7'], 'linux': ['lucid'], 'android': ['icecreamsandwich'], @@ -111,6 +112,13 @@ class ChromiumPort(Port): def is_chromium(self): return True + def default_max_locked_shards(self): + """Return the number of "locked" shards to run in parallel (like the http tests).""" + max_locked_shards = int(self.default_child_processes()) / 4 + if not max_locked_shards: + return 1 + return max_locked_shards + def default_pixel_tests(self): return True @@ -332,13 +340,16 @@ class ChromiumPort(Port): 'win_layout_rel', ]) + def warn_if_bug_missing_in_test_expectations(self): + return True + def expectations_files(self): paths = [self.path_to_test_expectations_file()] skia_expectations_path = self.path_from_chromium_base('skia', 'skia_test_expectations.txt') + # FIXME: we should probably warn if this file is missing in some situations. + # See the discussion in webkit.org/b/97699. if self._filesystem.exists(skia_expectations_path): paths.append(skia_expectations_path) - else: - _log.warning("Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky.") builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME') if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names: @@ -376,6 +387,9 @@ class ChromiumPort(Port): VirtualTestSuite('platform/chromium/virtual/gpu/fast/hidpi', 'fast/hidpi', ['--force-compositing-mode']), + VirtualTestSuite('platform/chromium/virtual/softwarecompositing', + 'compositing', + ['--enable-software-compositing']), ] # diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py index 3dfeab7a3..e246b8870 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py @@ -77,45 +77,51 @@ TEST_PATH_PREFIX = '/all-tests' FORWARD_PORTS = '8000 8080 8443 8880 9323' MS_TRUETYPE_FONTS_DIR = '/usr/share/fonts/truetype/msttcorefonts/' +MS_TRUETYPE_FONTS_PACKAGE = 'ttf-mscorefonts-installer' # Timeout in seconds to wait for start/stop of DumpRenderTree. DRT_START_STOP_TIMEOUT_SECS = 10 -# List of fonts that layout tests expect, copied from DumpRenderTree/gtk/TestShellX11.cpp. +# List of fonts that layout tests expect, copied from DumpRenderTree/chromium/TestShellX11.cpp. HOST_FONT_FILES = [ - [MS_TRUETYPE_FONTS_DIR, 'Arial.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Comic_Sans_MS.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Comic_Sans_MS_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Impact.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Italic.ttf'], + [[MS_TRUETYPE_FONTS_DIR], 'Arial.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Impact.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], # The Microsoft font EULA - ['/usr/share/doc/ttf-mscorefonts-installer/', 'READ_ME!.gz'], - ['/usr/share/fonts/truetype/ttf-dejavu/', 'DejaVuSans.ttf'], + [['/usr/share/doc/ttf-mscorefonts-installer/'], 'READ_ME!.gz', MS_TRUETYPE_FONTS_PACKAGE], + # Other fonts: Arabic, CJK, Indic, Thai, etc. + [['/usr/share/fonts/truetype/ttf-dejavu/'], 'DejaVuSans.ttf', 'ttf-dejavu'], + [['/usr/share/fonts/truetype/kochi/'], 'kochi-mincho.ttf', 'ttf-kochi-mincho'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_hi.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_ta.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'MuktiNarrow.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/thai/', '/usr/share/fonts/truetype/tlwg/'], 'Garuda.ttf', 'fonts-tlwg-garuda'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/', '/usr/share/fonts/truetype/ttf-punjabi-fonts/'], 'lohit_pa.ttf', 'ttf-indic-fonts-core'], ] -# Should increase this version after changing HOST_FONT_FILES. -FONT_FILES_VERSION = 2 DEVICE_FONTS_DIR = DEVICE_DRT_DIR + 'fonts/' @@ -198,7 +204,8 @@ class ChromiumAndroidPort(chromium.ChromiumPort): def check_build(self, needs_http): result = super(ChromiumAndroidPort, self).check_build(needs_http) - result = self.check_wdiff() and result + result = self._check_file_exists(self._path_to_md5sum(), 'md5sum utility') and result + result = self._check_file_exists(self._path_to_forwarder(), 'forwarder utility') and result if not result: _log.error('For complete Android build requirements, please see:') _log.error('') @@ -207,11 +214,15 @@ class ChromiumAndroidPort(chromium.ChromiumPort): return result def check_sys_deps(self, needs_http): - for (font_dir, font_file) in HOST_FONT_FILES: - font_path = font_dir + font_file - if not self._check_file_exists(font_path, 'font file'): - _log.error('You are missing %s. Try installing msttcorefonts. ' - 'See build instructions.' % font_path) + for (font_dirs, font_file, package) in HOST_FONT_FILES: + exists = False + for font_dir in font_dirs: + font_path = font_dir + font_file + if self._check_file_exists(font_path, '', logging=False): + exists = True + break + if not exists: + _log.error('You are missing %s under %s. Try installing %s. See build instructions.' % (font_file, font_dirs, package)) return False return True @@ -271,6 +282,9 @@ class ChromiumAndroidPort(chromium.ChromiumPort): def _path_to_forwarder(self): return self._build_path('forwarder') + def _path_to_md5sum(self): + return self._build_path(MD5SUM_DEVICE_FILE_NAME) + def _path_to_image_diff(self): return self._host_port._path_to_image_diff() @@ -330,12 +344,10 @@ class ChromiumAndroidDriver(driver.Driver): super(ChromiumAndroidDriver, self).__del__() def _setup_md5sum_and_push_data_if_needed(self): - self._md5sum_path = self._port._build_path_with_configuration(self._port.get_option('configuration'), MD5SUM_DEVICE_FILE_NAME) - assert os.path.exists(self._md5sum_path) - + self._md5sum_path = self._port._path_to_md5sum() if not self._file_exists_on_device(MD5SUM_DEVICE_PATH): if not self._push_to_device(self._md5sum_path, MD5SUM_DEVICE_PATH): - _log.error('Could not push md5sum to device') + raise AssertionError('Could not push md5sum to device') self._push_executable() self._push_fonts() @@ -401,31 +413,20 @@ class ChromiumAndroidDriver(driver.Driver): self._abort('Failed to install %s onto device: %s' % (drt_host_path, install_result)) def _push_fonts(self): - if not self._check_version(DEVICE_FONTS_DIR, FONT_FILES_VERSION): - self._log_debug('Pushing fonts') - path_to_ahem_font = self._port._build_path('AHEM____.TTF') - self._push_to_device(path_to_ahem_font, DEVICE_FONTS_DIR + 'AHEM____.TTF') - for (host_dir, font_file) in HOST_FONT_FILES: - self._push_to_device(host_dir + font_file, DEVICE_FONTS_DIR + font_file) - self._link_device_file('/system/fonts/DroidSansFallback.ttf', DEVICE_FONTS_DIR + 'DroidSansFallback.ttf') - self._update_version(DEVICE_FONTS_DIR, FONT_FILES_VERSION) + self._log_debug('Pushing fonts') + path_to_ahem_font = self._port._build_path('AHEM____.TTF') + self._push_file_if_needed(path_to_ahem_font, DEVICE_FONTS_DIR + 'AHEM____.TTF') + for (host_dirs, font_file, package) in HOST_FONT_FILES: + for host_dir in host_dirs: + host_font_path = host_dir + font_file + if self._port._check_file_exists(host_font_path, '', logging=False): + self._push_file_if_needed(host_font_path, DEVICE_FONTS_DIR + font_file) def _push_test_resources(self): self._log_debug('Pushing test resources') for resource in TEST_RESOURCES_TO_PUSH: self._push_file_if_needed(self._port.layout_tests_dir() + '/' + resource, DEVICE_LAYOUT_TESTS_DIR + resource) - def _check_version(self, dir, version): - assert(dir.endswith('/')) - try: - device_version = int(self._run_adb_command(['shell', 'cat %sVERSION || echo 0' % dir])) - return device_version == version - except: - return False - - def _update_version(self, dir, version): - self._run_adb_command(['shell', 'echo %d > %sVERSION' % (version, dir)]) - def _run_adb_command(self, cmd, ignore_error=False): self._log_debug('Run adb command: ' + str(cmd)) if ignore_error: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py index 2ffb77979..bb4229e65 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py @@ -30,6 +30,7 @@ import optparse import StringIO import time import unittest +import sys from webkitpy.common.system import executive_mock from webkitpy.common.system.executive_mock import MockExecutive2 @@ -244,7 +245,8 @@ class ChromiumAndroidDriverTest(unittest.TestCase): def test_command_from_driver_input(self): driver_input = driver.DriverInput('foo/bar/test.html', 10, 'checksum', True) expected_command = "/data/local/tmp/third_party/WebKit/LayoutTests/foo/bar/test.html'--pixel-test'checksum\n" - self.assertEquals(self.driver._command_from_driver_input(driver_input), expected_command) + if (sys.platform != "cygwin"): + self.assertEquals(self.driver._command_from_driver_input(driver_input), expected_command) driver_input = driver.DriverInput('http/tests/foo/bar/test.html', 10, 'checksum', True) expected_command = "http://127.0.0.1:8000/foo/bar/test.html'--pixel-test'checksum\n" diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index fa8c274ea..a2252c1b3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -155,14 +155,6 @@ class ChromiumLinuxPort(chromium.ChromiumPort): else: return '/usr/sbin/apache2' - def _path_to_apache_config_file(self): - if self._is_redhat_based(): - config_name = 'fedora-httpd.conf' - else: - config_name = 'apache2-debian-httpd.conf' - - return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_name) - def _path_to_lighttpd(self): return "/usr/sbin/lighttpd" @@ -178,12 +170,3 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _path_to_helper(self): return None - - def _path_to_wdiff(self): - if self._is_redhat_based(): - return '/usr/bin/dwdiff' - else: - return '/usr/bin/wdiff' - - def _is_redhat_based(self): - return self._filesystem.exists(self._filesystem.join('/etc', 'redhat-release')) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py index fb90d1b9b..a082a131c 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py @@ -48,6 +48,13 @@ class ChromiumPortTestCase(port_testcase.PortTestCase): port = self.make_port() port.check_build(needs_http=True) + def test_default_max_locked_shards(self): + port = self.make_port() + port.default_child_processes = lambda: 16 + self.assertEquals(port.default_max_locked_shards(), 4) + port.default_child_processes = lambda: 2 + self.assertEquals(port.default_max_locked_shards(), 1) + def test_default_timeout_ms(self): self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000) self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 12000) @@ -72,6 +79,8 @@ class ChromiumPortTestCase(port_testcase.PortTestCase): TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('lion', 'x86', 'debug'), TestConfiguration('lion', 'x86', 'release'), + TestConfiguration('mountainlion', 'x86', 'debug'), + TestConfiguration('mountainlion', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug'), TestConfiguration('xp', 'x86', 'release'), TestConfiguration('win7', 'x86', 'debug'), diff --git a/Tools/Scripts/webkitpy/layout_tests/port/driver.py b/Tools/Scripts/webkitpy/layout_tests/port/driver.py index 4bf53d46a..7993d0577 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/driver.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/driver.py @@ -76,7 +76,7 @@ class DriverOutput(object): strip_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled')) def __init__(self, text, image, image_hash, audio, crash=False, - test_time=0, timeout=False, error='', crashed_process_name='??', + test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??', crashed_pid=None, crash_log=None): # FIXME: Args could be renamed to better clarify what they do. self.text = text @@ -89,6 +89,7 @@ class DriverOutput(object): self.crashed_pid = crashed_pid self.crash_log = crash_log self.test_time = test_time + self.measurements = measurements self.timeout = timeout self.error = error # stderr output @@ -138,6 +139,8 @@ class Driver(object): self.err_seen_eof = False self._server_process = None + self._measurements = {} + def __del__(self): self.stop() @@ -193,7 +196,7 @@ class Driver(object): crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test) return DriverOutput(text, image, actual_image_hash, audio, - crash=crashed, test_time=time.time() - test_begin_time, + crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements, timeout=timed_out, error=self.error_from_test, crashed_process_name=self._crashed_process_name, crashed_pid=self._crashed_pid, crash_log=crash_log) @@ -360,6 +363,10 @@ class Driver(object): def _read_first_block(self, deadline): # returns (text_content, audio_content) block = self._read_block(deadline) + if block.malloc: + self._measurements['Malloc'] = float(block.malloc) + if block.js_heap: + self._measurements['JSHeap'] = float(block.js_heap) if block.content_type == 'audio/wav': return (None, block.decoded_content) return (block.decoded_content, None) @@ -384,7 +391,9 @@ class Driver(object): if (self._read_header(block, line, 'Content-Type: ', 'content_type') or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding') or self._read_header(block, line, 'Content-Length: ', '_content_length', int) - or self._read_header(block, line, 'ActualHash: ', 'content_hash')): + or self._read_header(block, line, 'ActualHash: ', 'content_hash') + or self._read_header(block, line, 'DumpMalloc: ', 'malloc') + or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')): return # Note, we're not reading ExpectedHash: here, but we could. # If the line wasn't a header, we just append it to the content. @@ -450,6 +459,8 @@ class ContentBlock(object): # Content is treated as binary data even though the text output is usually UTF-8. self.content = str() # FIXME: Should be bytearray() once we require Python 2.6. self.decoded_content = None + self.malloc = None + self.js_heap = None def decode_content(self): if self.encoding == 'base64' and self.content is not None: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/efl.py b/Tools/Scripts/webkitpy/layout_tests/port/efl.py index 269146aed..1022cd7b7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/efl.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/efl.py @@ -32,7 +32,7 @@ import os from webkitpy.layout_tests.models.test_configuration import TestConfiguration from webkitpy.layout_tests.port.base import Port from webkitpy.layout_tests.port.pulseaudio_sanitizer import PulseAudioSanitizer - +from webkitpy.layout_tests.port.xvfbdriver import XvfbDriver class EflPort(Port, PulseAudioSanitizer): port_name = 'efl' @@ -80,6 +80,9 @@ class EflPort(Port, PulseAudioSanitizer): def _generate_all_test_configurations(self): return [TestConfiguration(version=self._version, architecture='x86', build_type=build_type) for build_type in self.ALL_BUILD_TYPES] + def _driver_class(self): + return XvfbDriver + def _path_to_driver(self): return self._build_path('bin', self.driver_name()) @@ -98,18 +101,22 @@ class EflPort(Port, PulseAudioSanitizer): search_paths = [] if self.get_option('webkit_test_runner'): search_paths.append(self.port_name + '-wk2') + search_paths.append('wk2') else: search_paths.append(self.port_name + '-wk1') search_paths.append(self.port_name) return search_paths def expectations_files(self): + # FIXME: We should be able to use the default algorithm here. return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self._search_paths()])) def show_results_html_file(self, results_filename): # FIXME: We should find a way to share this implmentation with Gtk, # or teach run-launcher how to call run-safari and move this down to WebKitPort. run_launcher_args = ["file://%s" % results_filename] + if self.get_option('webkit_test_runner'): + run_launcher_args.append('-2') # FIXME: old-run-webkit-tests also added ["-graphicssystem", "raster", "-style", "windows"] # FIXME: old-run-webkit-tests converted results_filename path for cygwin. self._run_script("run-launcher", run_launcher_args) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index bb077c40a..2980c2d23 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -105,7 +105,7 @@ class FactoryTest(unittest.TestCase): self.assertRaises(NotImplementedError, factory.PortFactory(MockSystemHost(os_name='vms')).get) def test_get_from_builder_name(self): - self.assertEquals(factory.PortFactory(MockSystemHost()).get_from_builder_name('Webkit Mac10.7').name(), + self.assertEquals(factory.PortFactory(MockSystemHost()).get_from_builder_name('WebKit Mac10.7').name(), 'chromium-mac-lion') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py index f02d14819..6c57b5363 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -41,6 +41,9 @@ class GtkPort(Port, PulseAudioSanitizer): def expectations_files(self): return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in self._skipped_file_search_paths()] + def warn_if_bug_missing_in_test_expectations(self): + return True + def _port_flag_for_scripts(self): return "--gtk" @@ -89,18 +92,6 @@ class GtkPort(Port, PulseAudioSanitizer): def _path_to_image_diff(self): return self._build_path('Programs', 'ImageDiff') - def _path_to_apache(self): - if self._is_redhat_based(): - return '/usr/sbin/httpd' - else: - return '/usr/sbin/apache2' - - def _path_to_wdiff(self): - if self._is_redhat_based(): - return '/usr/bin/dwdiff' - else: - return '/usr/bin/wdiff' - def _path_to_webcore_library(self): gtk_library_names = [ "libwebkitgtk-1.0.so", diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 985241ede..f704a7a13 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -30,6 +30,7 @@ import errno import logging +import os import socket import sys import time @@ -89,6 +90,13 @@ class PortTestCase(unittest.TestCase): port_name = self.port_maker.determine_full_port_name(host, options, port_name) return self.port_maker(host, port_name, options=options, config=config, **kwargs) + def test_default_max_locked_shards(self): + port = self.make_port() + port.default_child_processes = lambda: 16 + self.assertEquals(port.default_max_locked_shards(), 1) + port.default_child_processes = lambda: 2 + self.assertEquals(port.default_max_locked_shards(), 1) + def test_default_timeout_ms(self): self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 35000) self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 35000) @@ -582,10 +590,29 @@ class PortTestCase(unittest.TestCase): def test_path_to_apache_config_file(self): port = TestWebKitPort() + + saved_environ = os.environ.copy() + try: + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/path/to/httpd.conf' + self.assertRaises(IOError, port._path_to_apache_config_file) + port._filesystem.write_text_file('/existing/httpd.conf', 'Hello, world!') + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' + self.assertEquals(port._path_to_apache_config_file(), '/existing/httpd.conf') + finally: + os.environ = saved_environ.copy() + # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value. port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf' self.assertEquals(port._path_to_apache_config_file(), '/mock-checkout/LayoutTests/http/conf/httpd.conf') + # Check that even if we mock out _apache_config_file_name, the environment variable takes precedence. + saved_environ = os.environ.copy() + try: + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' + self.assertEquals(port._path_to_apache_config_file(), '/existing/httpd.conf') + finally: + os.environ = saved_environ.copy() + def test_check_build(self): port = self.make_port(options=MockOptions(build=True)) self.build_called = False diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt.py b/Tools/Scripts/webkitpy/layout_tests/port/qt.py index 828a80d64..5b8342d9c 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt.py @@ -88,9 +88,9 @@ class QtPort(Port): def _path_to_webcore_library(self): if self.operating_system() == 'mac': - return self._build_path('lib/QtWebKit.framework/QtWebKit') + return self._build_path('lib/QtWebKitWidgets.framework/QtWebKitWidgets') else: - return self._build_path('lib/libQtWebKit.so') + return self._build_path('lib/libQtWebKitWidgets.so') def _modules_to_search_for_symbols(self): # We search in every library to be reliable in the case of building with CONFIG+=force_static_libs_as_shared. @@ -135,23 +135,21 @@ class QtPort(Port): search_paths.append('qt-4.8') elif version: search_paths.append('qt-5.0') - search_paths.append(self.port_name + '-' + self.host.platform.os_name) + search_paths.append(self.port_name + '-' + self.operating_system()) search_paths.append(self.port_name) return search_paths def default_baseline_search_path(self): return map(self._webkit_baseline_path, self._search_paths()) - def _skipped_file_search_paths(self): - skipped_path = self._search_paths() - if self.get_option('webkit_test_runner') and '5.0' in self.qt_version(): - skipped_path.append('wk2') - return skipped_path - def expectations_files(self): + paths = self._search_paths() + if self.get_option('webkit_test_runner'): + paths.append('wk2') + # expectations_files() uses the directories listed in _search_paths reversed. # e.g. qt -> qt-linux -> qt-4.8 - return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self._search_paths()])) + return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in paths])) def setup_environ_for_server(self, server_name=None): clean_env = super(QtPort, self).setup_environ_for_server(server_name) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py index 374b10270..cf09bd8e0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py @@ -71,14 +71,6 @@ class QtPortTest(port_testcase.PortTestCase): absolute_search_paths = map(port._webkit_baseline_path, search_paths) self.assertEquals(port.baseline_search_path(), absolute_search_paths) - def _assert_skipped_path(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): - host = MockSystemHost(os_name=os_name) - host.executive = MockExecutive2(self._qt_version(qt_version)) - port_name = 'qt-' + os_name - port = self.make_port(host=host, qt_version=qt_version, port_name=port_name, - options=MockOptions(webkit_test_runner=use_webkit2, platform='qt')) - self.assertEquals(port._skipped_file_search_paths(), search_paths) - def _assert_expectations_files(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): # FIXME: Port constructors should not "parse" the port name, but # rather be passed components (directly or via setters). Once @@ -100,19 +92,13 @@ class QtPortTest(port_testcase.PortTestCase): for case in self.search_paths_cases: self._assert_search_path(**case) - def test_skipped_file_search_path(self): - caselist = self.search_paths_cases[:] - for case in caselist: - if case['use_webkit2'] and case['qt_version'] == '5.0': - case['search_paths'].append("wk2") - self._assert_skipped_path(**case) - def test_expectations_files(self): for case in self.search_paths_cases: expectations_case = deepcopy(case) - expectations_case['search_paths'] = [] - for path in reversed(case['search_paths']): - expectations_case['search_paths'].append('/mock-checkout/LayoutTests/platform/%s/TestExpectations' % (path)) + if expectations_case['use_webkit2']: + expectations_case['search_paths'].append("wk2") + expectations_case['search_paths'].reverse() + expectations_case['search_paths'] = map(lambda path: '/mock-checkout/LayoutTests/platform/%s/TestExpectations' % (path), expectations_case['search_paths']) self._assert_expectations_files(**expectations_case) def test_show_results_html_file(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py index bfa6aab80..bfdf8301b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -84,6 +84,17 @@ class ServerProcess(object): return self._pid def _reset(self): + if getattr(self, '_proc', None): + if self._proc.stdin: + self._proc.stdin.close() + self._proc.stdin = None + if self._proc.stdout: + self._proc.stdout.close() + self._proc.stdout = None + if self._proc.stderr: + self._proc.stderr.close() + self._proc.stderr = None + self._proc = None self._output = str() # bytesarray() once we require Python 2.6 self._error = str() # bytesarray() once we require Python 2.6 @@ -321,8 +332,11 @@ class ServerProcess(object): now = time.time() self._proc.stdin.close() + self._proc.stdin = None + killed = False if not timeout_secs: self._kill() + killed = True elif not self._host.platform.is_win(): # FIXME: Why aren't we calling this on win? deadline = now + timeout_secs @@ -331,13 +345,15 @@ class ServerProcess(object): if self._proc.poll() is None: _log.warning('stopping %s timed out, killing it' % self._name) self._kill() + killed = True _log.warning('killed') # read any remaining data on the pipes and return it. - if self._use_win32_apis: - self._wait_for_data_and_update_buffers_using_win32_apis(now) - else: - self._wait_for_data_and_update_buffers_using_select(now, stopping=True) + if not killed: + if self._use_win32_apis: + self._wait_for_data_and_update_buffers_using_win32_apis(now) + else: + self._wait_for_data_and_update_buffers_using_select(now, stopping=True) out, err = self._output, self._error self._reset() return (out, err) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py index 48c41e6f2..7a5ac45d4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py @@ -57,6 +57,7 @@ class TrivialMockPort(object): class MockFile(object): def __init__(self, server_process): self._server_process = server_process + self.closed = False def fileno(self): return 1 @@ -66,7 +67,7 @@ class MockFile(object): raise IOError def close(self): - pass + self.closed = True class MockProc(object): @@ -87,6 +88,8 @@ class FakeServerProcess(server_process.ServerProcess): def _start(self): self._proc = MockProc(self) self.stdin = self._proc.stdin + self.stdout = self._proc.stdout + self.stderr = self._proc.stderr self._pid = self._proc.pid self.broken_pipes = [] @@ -121,6 +124,15 @@ class TestServerProcess(unittest.TestCase): proc.stop(0) + def test_cleanup(self): + port_obj = TrivialMockPort() + server_process = FakeServerProcess(port_obj=port_obj, name="test", cmd=["test"]) + server_process._start() + server_process.stop() + self.assertTrue(server_process.stdin.closed) + self.assertTrue(server_process.stdout.closed) + self.assertTrue(server_process.stderr.closed) + def test_broken_pipe(self): port_obj = TrivialMockPort() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py index d008d995d..726575614 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -239,6 +239,11 @@ layer at (0,0) size 800x34 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') + # For testing that virtual test suites don't expand names containing themselves + # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). + tests.add('passes/test-virtual-passes.html') + tests.add('passes/passes/test-virtual-passes.html') + return tests diff --git a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py index 7e386a108..b927720db 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py @@ -28,41 +28,60 @@ import logging import os +import re +import time from webkitpy.layout_tests.port.server_process import ServerProcess from webkitpy.layout_tests.port.driver import Driver +from webkitpy.common.system.file_lock import FileLock _log = logging.getLogger(__name__) class XvfbDriver(Driver): - def _start(self, pixel_tests, per_test_args): - - # Collect the number of X servers running already and make - # sure our Xvfb process doesn't clash with any of them. - def x_filter(process_name): - return process_name.find("Xorg") > -1 + def __init__(self, *args, **kwargs): + Driver.__init__(self, *args, **kwargs) + self._guard_lock = None + self._startup_delay_secs = 1.0 - running_displays = len(self._port.host.executive.running_pids(x_filter)) + def _next_free_display(self): + running_pids = self._port.host.executive.run_command(['ps', '-eo', 'comm,command']) + reserved_screens = set() + for pid in running_pids.split('\n'): + match = re.match('(X|Xvfb|Xorg)\s+.*\s:(?P<screen_number>\d+)', pid) + if match: + reserved_screens.add(int(match.group('screen_number'))) + for i in range(99): + if i not in reserved_screens: + _guard_lock_file = self._port.host.filesystem.join('/tmp', 'WebKitXvfb.lock.%i' % i) + self._guard_lock = FileLock(_guard_lock_file) + if self._guard_lock.acquire_lock(): + return i + def _start(self, pixel_tests, per_test_args): # Use even displays for pixel tests and odd ones otherwise. When pixel tests are disabled, # DriverProxy creates two drivers, one for normal and the other for ref tests. Both have # the same worker number, so this prevents them from using the same Xvfb instance. - display_id = self._worker_number * 2 + running_displays - if pixel_tests: - display_id += 1 + display_id = self._next_free_display() self._lock_file = "/tmp/.X%d-lock" % display_id run_xvfb = ["Xvfb", ":%d" % display_id, "-screen", "0", "800x600x24", "-nolisten", "tcp"] with open(os.devnull, 'w') as devnull: self._xvfb_process = self._port.host.executive.popen(run_xvfb, stderr=devnull) + # Crashes intend to occur occasionally in the first few tests that are run through each + # worker because the Xvfb display isn't ready yet. Halting execution a bit should avoid that. + time.sleep(self._startup_delay_secs) + server_name = self._port.driver_name() environment = self._port.setup_environ_for_server(server_name) # We must do this here because the DISPLAY number depends on _worker_number environment['DISPLAY'] = ":%d" % display_id # Drivers should use separate application cache locations environment['XDG_CACHE_HOME'] = self._port.host.filesystem.join(self._port.results_directory(), '%s-appcache-%d' % (server_name, self._worker_number)) + self._driver_tempdir = self._port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name()) + environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir) + environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir() self._crashed_process_name = None self._crashed_pid = None @@ -71,6 +90,9 @@ class XvfbDriver(Driver): def stop(self): super(XvfbDriver, self).stop() + if self._guard_lock: + self._guard_lock.release_lock() + self._guard_lock = None if getattr(self, '_xvfb_process', None): self._port.host.executive.kill_process(self._xvfb_process.pid) self._xvfb_process = None diff --git a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py index 37a2fbd4d..220dd3517 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py @@ -30,6 +30,7 @@ import unittest from webkitpy.common.system.deprecated_logging import log from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.executive_mock import MockExecutive2 from webkitpy.common.system.outputcapture import OutputCapture from webkitpy.common.system.systemhost_mock import MockSystemHost from webkitpy.layout_tests.port import Port @@ -39,13 +40,14 @@ from webkitpy.layout_tests.port.xvfbdriver import XvfbDriver class XvfbDriverTest(unittest.TestCase): - def make_driver(self, worker_number=0, xorg_running=False): - port = Port(host=MockSystemHost(log_executive=True), config=MockConfig()) + def make_driver(self, worker_number=0, xorg_running=False, executive=None): + port = Port(host=MockSystemHost(log_executive=True, executive=executive), config=MockConfig()) port._server_process_constructor = MockServerProcess if xorg_running: port._executive._running_pids['Xorg'] = 108 driver = XvfbDriver(port, worker_number=worker_number, pixel_tests=True) + driver._startup_delay_secs = 0 return driver def cleanup_driver(self, driver): @@ -61,26 +63,54 @@ class XvfbDriverTest(unittest.TestCase): def test_start_no_pixel_tests(self): driver = self.make_driver() - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0") self.cleanup_driver(driver) def test_start_pixel_tests(self): driver = self.make_driver() - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':1', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":1", pixel_tests=True) + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) self.cleanup_driver(driver) def test_start_arbitrary_worker_number(self): driver = self.make_driver(worker_number=17) - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':35', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":35", pixel_tests=True) + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) self.cleanup_driver(driver) - def test_start_existing_xorg_process(self): - driver = self.make_driver(xorg_running=True) - expected_stderr = "MOCK running_pids: [108]\nMOCK popen: ['Xvfb', ':1', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":1") + def disabled_test_next_free_display(self): + output = "Xorg /usr/bin/X :0 -auth /var/run/lightdm/root/:0 -nolisten tcp vt7 -novtswitch -background none\nXvfb Xvfb :1 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 2) + self.cleanup_driver(driver) + output = "X /usr/bin/X :0 vt7 -nolisten tcp -auth /var/run/xauth/A:0-8p7Ybb" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 1) + self.cleanup_driver(driver) + output = "Xvfb Xvfb :0 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 1) + self.cleanup_driver(driver) + output = "Xvfb Xvfb :1 -screen 0 800x600x24 -nolisten tcp\nXvfb Xvfb :0 -screen 0 800x600x24 -nolisten tcp\nXvfb Xvfb :3 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 2) + self.cleanup_driver(driver) + + def test_start_next_worker(self): + driver = self.make_driver() + driver._next_free_display = lambda: 0 + expected_stderr = "MOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) + self.cleanup_driver(driver) + driver = self.make_driver() + driver._next_free_display = lambda: 3 + expected_stderr = "MOCK popen: ['Xvfb', ':3', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":3", pixel_tests=True) self.cleanup_driver(driver) def test_stop(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index e784cb61d..89522079c 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -129,6 +129,9 @@ def _set_up_derived_options(port, options): if not options.child_processes: options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", str(port.default_child_processes())) + if not options.max_locked_shards: + options.max_locked_shards = int(os.environ.get("WEBKIT_TEST_MAX_LOCKED_SHARDS", + str(port.default_max_locked_shards()))) if not options.configuration: options.configuration = port.default_configuration() @@ -375,7 +378,11 @@ def parse_args(args=None): optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option("--skipped", action="store", default="default", - help="control how tests marked SKIP are run. 'default' == Skip, 'ignore' == Run them anyway, 'only' == only run the SKIP tests."), + help=("control how tests marked SKIP are run. " + "'default' == Skip tests unless explicitly listed on the command line, " + "'ignore' == Run them anyway, " + "'only' == only run the SKIP tests, " + "'always' == always skip, even if listed on the command line.")), optparse.make_option("--force", dest="skipped", action="store_const", const='ignore', help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"), optparse.make_option("--time-out-ms", @@ -412,7 +419,7 @@ def parse_args(args=None): optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), - optparse.make_option("--max-locked-shards", type="int", default=1, + optparse.make_option("--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option("--additional-env-var", type="string", action="append", default=[], help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"), diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py index 4afcc1466..85437449b 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py @@ -301,6 +301,12 @@ class MainTest(unittest.TestCase, StreamTestingMixin): for batch in batch_tests_run: self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch)) + def test_max_locked_shards(self): + if not self.should_test_processes: + return + _, _, regular_output, _ = logging_run(['--debug-rwt-logging', '--child-processes', '2'], shared_port=False) + self.assertTrue(any(['(1 locked)' in line for line in regular_output.buflist])) + def test_child_processes_2(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( @@ -310,7 +316,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): def test_child_processes_min(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( - ['--debug-rwt-logging', '--child-processes', '2', 'passes'], + ['--debug-rwt-logging', '--child-processes', '2', '-i', 'passes/passes', 'passes'], tests_included=True, shared_port=False) self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist])) @@ -418,6 +424,10 @@ class MainTest(unittest.TestCase, StreamTestingMixin): self.assertEquals(get_tests_run(['--skipped=only', 'passes'], tests_included=True, flatten_batches=True), ['passes/skipped/skip.html']) + # Now check that we don't run anything. + self.assertEquals(get_tests_run(['--skipped=always', 'passes/skipped/skip.html'], tests_included=True, flatten_batches=True), + []) + def test_iterations(self): tests_to_run = ['passes/image.html', 'passes/text.html'] tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True) @@ -775,7 +785,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): # These next tests test that we run the tests in ascending alphabetical # order per directory. HTTP tests are sharded separately from other tests, # so we have to test both. - tests_run = get_tests_run(['passes'], tests_included=True, flatten_batches=True) + tests_run = get_tests_run(['-i', 'passes/passes', 'passes'], tests_included=True, flatten_batches=True) self.assertEquals(tests_run, sorted(tests_run)) tests_run = get_tests_run(['http/tests/passes'], tests_included=True, flatten_batches=True) @@ -922,6 +932,11 @@ class MainTest(unittest.TestCase, StreamTestingMixin): # child process (e.g., on win32) and we need to make sure that works and we still # see the verbose log output. However, we can't use logging_run() because using # outputcapture to capture stdout and stderr latter results in a nonpicklable host. + + # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559 + if not self.should_test_processes: + return + options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) host = MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) diff --git a/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py b/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py index 7cd4af538..a616fab5b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py +++ b/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py @@ -54,7 +54,6 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): self._name = 'httpd' self._mappings = [{'port': 8000}, {'port': 8080}, - {'port': 8081}, {'port': 8443, 'sslcert': True}] self._output_dir = output_dir self._filesystem.maybe_make_directory(output_dir) @@ -79,7 +78,6 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir, '-c', "\'Alias /media-resources \"%s\"'" % media_resources_dir, '-C', "\'Listen %s\'" % "127.0.0.1:8000", - '-C', "\'Listen %s\'" % "127.0.0.1:8081", '-c', "\'TypesConfig \"%s\"\'" % mime_types_path, '-c', "\'CustomLog \"%s\" common\'" % access_log, '-c', "\'ErrorLog \"%s\"\'" % error_log, diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest.py b/Tools/Scripts/webkitpy/performance_tests/perftest.py index fdac35b11..32b9d8bc6 100644 --- a/Tools/Scripts/webkitpy/performance_tests/perftest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftest.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (C) 2012 Google Inc. All rights reserved. +# Copyright (C) 2012 Zoltan Horvath, Adobe Systems Incorporated. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -202,11 +203,40 @@ class ChromiumStylePerfTest(PerfTest): class PageLoadingPerfTest(PerfTest): + _FORCE_GC_FILE = 'resources/force-gc.html' + def __init__(self, port, test_name, path_or_url): super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url) + self.force_gc_test = self._port.host.filesystem.join(self._port.perf_tests_dir(), self._FORCE_GC_FILE) + + def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False): + # Force GC to prevent pageload noise. See https://bugs.webkit.org/show_bug.cgi?id=98203 + super(PageLoadingPerfTest, self).run_single(driver, self.force_gc_test, time_out_ms, False) + return super(PageLoadingPerfTest, self).run_single(driver, path_or_url, time_out_ms, should_run_pixel_test) + + def calculate_statistics(self, values): + sorted_values = sorted(values) + + # Compute the mean and variance using Knuth's online algorithm (has good numerical stability). + squareSum = 0 + mean = 0 + for i, time in enumerate(sorted_values): + delta = time - mean + sweep = i + 1.0 + mean += delta / sweep + squareSum += delta * (time - mean) + + middle = int(len(sorted_values) / 2) + result = {'avg': mean, + 'min': sorted_values[0], + 'max': sorted_values[-1], + 'median': sorted_values[middle] if len(sorted_values) % 2 else (sorted_values[middle - 1] + sorted_values[middle]) / 2, + 'stdev': math.sqrt(squareSum / (len(sorted_values) - 1))} + return result def run(self, driver, time_out_ms): - test_times = [] + results = {} + results.setdefault(self.test_name(), {'unit': 'ms', 'values': []}) for i in range(0, 20): output = self.run_single(driver, self.path_or_url(), time_out_ms) @@ -214,30 +244,25 @@ class PageLoadingPerfTest(PerfTest): return None if i == 0: continue - test_times.append(output.test_time * 1000) - sorted_test_times = sorted(test_times) + results[self.test_name()]['values'].append(output.test_time * 1000) - # Compute the mean and variance using a numerically stable algorithm. - squareSum = 0 - mean = 0 - valueSum = sum(sorted_test_times) - for i, time in enumerate(sorted_test_times): - delta = time - mean - sweep = i + 1.0 - mean += delta / sweep - squareSum += delta * delta * (i / sweep) - - middle = int(len(test_times) / 2) - results = {'values': test_times, - 'avg': mean, - 'min': sorted_test_times[0], - 'max': sorted_test_times[-1], - 'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2, - 'stdev': math.sqrt(squareSum), - 'unit': 'ms'} - self.output_statistics(self.test_name(), results, '') - return {self.test_name(): results} + if not output.measurements: + continue + + for result_class, result in output.measurements.items(): + name = self.test_name() + ':' + result_class + if not name in results: + results.setdefault(name, {'values': []}) + results[name]['values'].append(result) + if result_class == 'Malloc' or result_class == 'JSHeap': + results[name]['unit'] = 'bytes' + + for result_class in results.keys(): + results[result_class].update(self.calculate_statistics(results[result_class]['values'])) + self.output_statistics(result_class, results[result_class], '') + + return results class ReplayServer(object): diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py index 27a4bb385..4410903e9 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py @@ -43,6 +43,10 @@ from webkitpy.performance_tests.perftest import PerfTestFactory from webkitpy.performance_tests.perftest import ReplayPerfTest +class MockPort(TestPort): + def __init__(self, custom_run_test=None): + super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test) + class MainTest(unittest.TestCase): def test_parse_output(self): output = DriverOutput('\n'.join([ @@ -98,39 +102,69 @@ class MainTest(unittest.TestCase): class TestPageLoadingPerfTest(unittest.TestCase): class MockDriver(object): - def __init__(self, values): + def __init__(self, values, test, measurements=None): self._values = values self._index = 0 + self._test = test + self._measurements = measurements def run_test(self, input, stop_when_done): + if input.test_name == self._test.force_gc_test: + return value = self._values[self._index] self._index += 1 if isinstance(value, str): return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value) else: - return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1]) + return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1], measurements=self._measurements) def test_run(self): - test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') - driver = TestPageLoadingPerfTest.MockDriver(range(1, 21)) + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test.run(driver, None), - {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms', + {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)]}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') - self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n') + self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n') + + def test_run_with_memory_output(self): + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + memory_results = {'Malloc': 10, 'JSHeap': 5} + self.maxDiff = None + driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) + output_capture = OutputCapture() + output_capture.capture_output() + try: + self.assertEqual(test.run(driver, None), + {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', + 'values': [i * 1000 for i in range(2, 21)]}, + 'some-test:Malloc': {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes', + 'values': [10] * 19}, + 'some-test:JSHeap': {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes', + 'values': [5] * 19}}) + finally: + actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() + self.assertEqual(actual_stdout, '') + self.assertEqual(actual_stderr, '') + self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n' + + 'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n' + + 'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n') def test_run_with_bad_output(self): output_capture = OutputCapture() output_capture.capture_output() try: - test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') - driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test) self.assertEqual(test.run(driver, None), None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() @@ -141,7 +175,7 @@ class TestPageLoadingPerfTest(unittest.TestCase): class TestReplayPerfTest(unittest.TestCase): - class ReplayTestPort(TestPort): + class ReplayTestPort(MockPort): def __init__(self, custom_run_test=None): class ReplayTestDriver(TestDriver): @@ -149,7 +183,7 @@ class TestReplayPerfTest(unittest.TestCase): return custom_run_test(text_input, stop_when_done) if custom_run_test else None self._custom_driver_class = ReplayTestDriver - super(self.__class__, self).__init__(host=MockHost()) + super(self.__class__, self).__init__() def _driver_class(self): return self._custom_driver_class @@ -179,6 +213,9 @@ class TestReplayPerfTest(unittest.TestCase): loaded_pages = [] def run_test(test_input, stop_when_done): + if test_input.test_name == test.force_gc_test: + loaded_pages.append(test_input) + return if test_input.test_name != "about:blank": self.assertEqual(test_input.test_name, 'http://some-test/') loaded_pages.append(test_input) @@ -196,8 +233,9 @@ class TestReplayPerfTest(unittest.TestCase): finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() - self.assertEqual(len(loaded_pages), 1) - self.assertEqual(loaded_pages[0].test_name, 'http://some-test/') + self.assertEqual(len(loaded_pages), 2) + self.assertEqual(loaded_pages[0].test_name, test.force_gc_test) + self.assertEqual(loaded_pages[1].test_name, 'http://some-test/') self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '') @@ -262,8 +300,9 @@ class TestReplayPerfTest(unittest.TestCase): finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() - self.assertEqual(len(loaded_pages), 1) - self.assertEqual(loaded_pages[0].test_name, 'http://some-test/') + self.assertEqual(len(loaded_pages), 2) + self.assertEqual(loaded_pages[0].test_name, test.force_gc_test) + self.assertEqual(loaded_pages[1].test_name, 'http://some-test/') self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n') @@ -316,15 +355,15 @@ class TestReplayPerfTest(unittest.TestCase): class TestPerfTestFactory(unittest.TestCase): def test_regular_test(self): - test = PerfTestFactory.create_perf_test(None, 'some-dir/some-test', '/path/some-dir/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') self.assertEqual(test.__class__, PerfTest) def test_inspector_test(self): - test = PerfTestFactory.create_perf_test(None, 'inspector/some-test', '/path/inspector/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test') self.assertEqual(test.__class__, ChromiumStylePerfTest) def test_page_loading_test(self): - test = PerfTestFactory.create_perf_test(None, 'PageLoad/some-test', '/path/PageLoad/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'PageLoad/some-test', '/path/PageLoad/some-test') self.assertEqual(test.__class__, PageLoadingPerfTest) diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py index c34d0b3e4..42e0d96e1 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py @@ -29,19 +29,17 @@ """Run Inspector's perf tests in perf mode.""" +import os import json import logging import optparse -import re -import sys import time from webkitpy.common import find_files +from webkitpy.common.checkout.scm.detection import SCMDetector from webkitpy.common.host import Host from webkitpy.common.net.file_uploader import FileUploader -from webkitpy.layout_tests.views import printing from webkitpy.performance_tests.perftest import PerfTestFactory -from webkitpy.performance_tests.perftest import ReplayPerfTest _log = logging.getLogger(__name__) @@ -65,7 +63,7 @@ class PerfTestsRunner(object): else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) - self._host._initialize_scm() + self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} @@ -73,6 +71,9 @@ class PerfTestsRunner(object): @staticmethod def _parse_args(args=None): + def _expand_path(option, opt_str, value, parser): + path = os.path.expandvars(os.path.expanduser(value)) + setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), @@ -98,15 +99,12 @@ class PerfTestsRunner(object): help="Pause before running the tests to let user attach a performance monitor."), optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page."), - optparse.make_option("--output-json-path", + optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists."), optparse.make_option("--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results."), - optparse.make_option("--slave-config-json-path", + optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file."), - optparse.make_option("--source-json-path", dest="slave_config_json_path", - # FIXME: Remove this option once build.webkit.org is updated to use --slave-config-json-path. - help="Deprecated. Overrides --slave-config-json-path."), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", @@ -181,11 +179,6 @@ class PerfTestsRunner(object): def _generate_and_show_results(self): options = self._options - if options.test_results_server: - # Remove this code once build.webkit.org started using --no-show-results and --reset-results - options.reset_results = True - options.show_results = False - output_json_path = self._output_json_path() output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) @@ -213,7 +206,8 @@ class PerfTestsRunner(object): if description: contents['description'] = description for (name, path) in self._port.repository_paths(): - contents[name + '-revision'] = self._host.scm().svn_revision(path) + scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() + contents[name + '-revision'] = scm.svn_revision(path) # FIXME: Add --branch or auto-detect the branch we're in for key, value in {'timestamp': int(timestamp), 'branch': self._default_branch, 'platform': platform, diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py index d3de7b3df..6119c61d3 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py @@ -473,7 +473,7 @@ max 548000 bytes def test_run_with_slave_config_json(self): runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', - '--source-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) + '--slave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value"}') self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) self.assertEqual(runner.load_output_json(), [{ @@ -625,8 +625,10 @@ max 548000 bytes '--builder-name', 'webkit-mac-1', '--build-number=56', '--time-out-ms=42', + '--no-show-results', + '--reset-results', '--output-json-path=a/output.json', - '--source-json-path=a/source.json', + '--slave-config-json-path=a/source.json', '--test-results-server=somehost', '--debug']) self.assertEqual(options.build, True) @@ -636,6 +638,8 @@ max 548000 bytes self.assertEqual(options.build_number, '56') self.assertEqual(options.time_out_ms, '42') self.assertEqual(options.configuration, 'Debug') + self.assertEqual(options.show_results, False) + self.assertEqual(options.reset_results, True) self.assertEqual(options.output_json_path, 'a/output.json') self.assertEqual(options.slave_config_json_path, 'a/source.json') self.assertEqual(options.test_results_server, 'somehost') diff --git a/Tools/Scripts/webkitpy/style/checker.py b/Tools/Scripts/webkitpy/style/checker.py index 5b11af483..9f27c36da 100644 --- a/Tools/Scripts/webkitpy/style/checker.py +++ b/Tools/Scripts/webkitpy/style/checker.py @@ -96,7 +96,6 @@ _BASE_FILTER_RULES = [ '-runtime/rtti', '-whitespace/blank_line', '-whitespace/end_of_line', - '-whitespace/labels', # List Python pep8 categories last. # # Because much of WebKit's Python code base does not abide by the diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp.py b/Tools/Scripts/webkitpy/style/checkers/cpp.py index a0051c979..ebbd1ad2f 100644 --- a/Tools/Scripts/webkitpy/style/checkers/cpp.py +++ b/Tools/Scripts/webkitpy/style/checkers/cpp.py @@ -2060,6 +2060,31 @@ def check_directive_indentation(clean_lines, line_number, file_state, error): error(line_number, 'whitespace/indent', 4, 'preprocessor directives (e.g., #ifdef, #define, #import) should never be indented.') +def get_initial_spaces_for_line(clean_line): + initial_spaces = 0 + while initial_spaces < len(clean_line) and clean_line[initial_spaces] == ' ': + initial_spaces += 1 + return initial_spaces + + +def check_indentation_amount(clean_lines, line_number, error): + line = clean_lines.elided[line_number] + initial_spaces = get_initial_spaces_for_line(line) + + if initial_spaces % 4: + error(line_number, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. Are you using a 4-space indent?') + return + + previous_line = get_previous_non_blank_line(clean_lines, line_number)[0] + if not previous_line.strip() or match(r'\s*\w+\s*:\s*$', previous_line) or previous_line[0] == '#': + return + + previous_line_initial_spaces = get_initial_spaces_for_line(previous_line) + if initial_spaces > previous_line_initial_spaces + 4: + error(line_number, 'whitespace/indent', 3, 'When wrapping a line, only indent 4 spaces.') + + def check_using_std(clean_lines, line_number, file_state, error): """Looks for 'using std::foo;' statements which should be replaced with 'using namespace std;'. @@ -2535,44 +2560,10 @@ def check_style(clean_lines, line_number, file_extension, class_state, file_stat error(line_number, 'whitespace/tab', 1, 'Tab found; better to use spaces') - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 4-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - initial_spaces = 0 cleansed_line = clean_lines.elided[line_number] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 if line and line[-1].isspace(): error(line_number, 'whitespace/end_of_line', 4, 'Line ends in whitespace. Consider deleting these extra spaces.') - # There are certain situations we allow one space, notably for labels - elif ((initial_spaces >= 1 and initial_spaces <= 3) - and not match(r'\s*\w+\s*:\s*$', cleansed_line)): - error(line_number, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 4-space indent?') - # Labels should always be indented at least one space. - elif not initial_spaces and line[:2] != '//': - label_match = match(r'(?P<label>[^:]+):\s*$', line) - - if label_match: - label = label_match.group('label') - # Only throw errors for stuff that is definitely not a goto label, - # because goto labels can in fact occur at the start of the line. - if label in ['public', 'private', 'protected'] or label.find(' ') != -1: - error(line_number, 'whitespace/labels', 4, - 'Labels should always be indented at least one space. ' - 'If this is a member-initializer list in a constructor, ' - 'the colon should be on the line after the definition header.') if (cleansed_line.count(';') > 1 # for loops are allowed two ;'s (and may run over two lines). @@ -2612,6 +2603,7 @@ def check_style(clean_lines, line_number, file_extension, class_state, file_stat check_check(clean_lines, line_number, error) check_for_comparisons_to_zero(clean_lines, line_number, error) check_for_null(clean_lines, line_number, file_state, error) + check_indentation_amount(clean_lines, line_number, error) _RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') @@ -3634,7 +3626,6 @@ class CppChecker(object): 'whitespace/end_of_line', 'whitespace/ending_newline', 'whitespace/indent', - 'whitespace/labels', 'whitespace/line_length', 'whitespace/newline', 'whitespace/operators', diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py index a1b91f292..6f001e0cb 100644 --- a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py +++ b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py @@ -1086,7 +1086,8 @@ class CppStyleTest(CppStyleTestBase): */ ''', '') self.assert_multi_line_lint( - r'''/* int a = 0; multi-liner + '''\ + /* int a = 0; multi-liner static const int b = 0;''', ['Could not find end of multi-line comment' ' [readability/multiline_comment] [5]', @@ -1125,97 +1126,111 @@ class CppStyleTest(CppStyleTestBase): def test_explicit_single_argument_constructors(self): # missing explicit is bad self.assert_multi_line_lint( - '''class Foo { - Foo(int f); - };''', + '''\ + class Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # missing explicit is bad, even with whitespace self.assert_multi_line_lint( - '''class Foo { - Foo (int f); - };''', + '''\ + class Foo { + Foo (int f); + };''', ['Extra space before ( in function call [whitespace/parens] [4]', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]']) # missing explicit, with distracting comment, is still bad self.assert_multi_line_lint( - '''class Foo { - Foo(int f); // simpler than Foo(blargh, blarg) - };''', + '''\ + class Foo { + Foo(int f); // simpler than Foo(blargh, blarg) + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # missing explicit, with qualified classname self.assert_multi_line_lint( - '''class Qualifier::AnotherOne::Foo { - Foo(int f); - };''', + '''\ + class Qualifier::AnotherOne::Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # structs are caught as well. self.assert_multi_line_lint( - '''struct Foo { - Foo(int f); - };''', + '''\ + struct Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # Templatized classes are caught as well. self.assert_multi_line_lint( - '''template<typename T> class Foo { - Foo(int f); - };''', + '''\ + template<typename T> class Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # proper style is okay self.assert_multi_line_lint( - '''class Foo { - explicit Foo(int f); - };''', + '''\ + class Foo { + explicit Foo(int f); + };''', '') # two argument constructor is okay self.assert_multi_line_lint( - '''class Foo { - Foo(int f, int b); - };''', + '''\ + class Foo { + Foo(int f, int b); + };''', '') # two argument constructor, across two lines, is okay self.assert_multi_line_lint( - '''class Foo { - Foo(int f, - int b); - };''', + '''\ + class Foo { + Foo(int f, + int b); + };''', '') # non-constructor (but similar name), is okay self.assert_multi_line_lint( - '''class Foo { - aFoo(int f); - };''', + '''\ + class Foo { + aFoo(int f); + };''', '') # constructor with void argument is okay self.assert_multi_line_lint( - '''class Foo { - Foo(void); - };''', + '''\ + class Foo { + Foo(void); + };''', '') # single argument method is okay self.assert_multi_line_lint( - '''class Foo { - Bar(int b); - };''', + '''\ + class Foo { + Bar(int b); + };''', '') # comments should be ignored self.assert_multi_line_lint( - '''class Foo { - // Foo(int f); - };''', + '''\ + class Foo { + // Foo(int f); + };''', '') # single argument function following class definition is okay # (okay, it's not actually valid, but we don't want a false positive) self.assert_multi_line_lint( - '''class Foo { - Foo(int f, int b); - }; - Foo(int f);''', + '''\ + class Foo { + Foo(int f, int b); + }; + Foo(int f);''', '') # single argument function is okay self.assert_multi_line_lint( @@ -1223,14 +1238,16 @@ class CppStyleTest(CppStyleTestBase): '') # single argument copy constructor is okay. self.assert_multi_line_lint( - '''class Foo { - Foo(const Foo&); - };''', + '''\ + class Foo { + Foo(const Foo&); + };''', '') self.assert_multi_line_lint( - '''class Foo { - Foo(Foo&); - };''', + '''\ + class Foo { + Foo(Foo&); + };''', '') def test_slash_star_comment_on_single_line(self): @@ -1248,9 +1265,6 @@ class CppStyleTest(CppStyleTestBase): ''' /*/ static Foo(int f);''', 'Could not find end of multi-line comment' ' [readability/multiline_comment] [5]') - self.assert_multi_line_lint( - ''' /**/ static Foo(int f);''', - '') # Test suspicious usage of "if" like this: # if (a == b) { @@ -1383,23 +1397,27 @@ class CppStyleTest(CppStyleTestBase): # or initializing an array self.assert_lint('int a[3] = { 1, 2, 3 };', '') self.assert_lint( - '''const int foo[] = - {1, 2, 3 };''', + '''\ + const int foo[] = + {1, 2, 3 };''', '') # For single line, unmatched '}' with a ';' is ignored (not enough context) self.assert_multi_line_lint( - '''int a[3] = { 1, - 2, - 3 };''', + '''\ + int a[3] = { 1, + 2, + 3 };''', '') self.assert_multi_line_lint( - '''int a[2][3] = { { 1, 2 }, - { 3, 4 } };''', + '''\ + int a[2][3] = { { 1, 2 }, + { 3, 4 } };''', '') self.assert_multi_line_lint( - '''int a[2][3] = - { { 1, 2 }, - { 3, 4 } };''', + '''\ + int a[2][3] = + { { 1, 2 }, + { 3, 4 } };''', '') # CHECK/EXPECT_TRUE/EXPECT_FALSE replacements @@ -1753,7 +1771,7 @@ class CppStyleTest(CppStyleTestBase): self.assert_lint('default:;', 'Semicolon defining empty statement. Use { } instead.' ' [whitespace/semicolon] [5]') - self.assert_lint(' ;', + self.assert_lint(' ;', 'Line contains only semicolon. If this should be an empty ' 'statement, use { } instead.' ' [whitespace/semicolon] [5]') @@ -1795,10 +1813,10 @@ class CppStyleTest(CppStyleTestBase): ' VeryLongNameType veryLongNameVariable) { }', '') self.assert_lint('template<>\n' 'string FunctionTemplateSpecialization<SomeType>(\n' - ' int x) { return ""; }', '') + ' int x) { return ""; }', '') self.assert_lint('template<>\n' 'string FunctionTemplateSpecialization<vector<A::B>* >(\n' - ' int x) { return ""; }', '') + ' int x) { return ""; }', '') # should not catch methods of template classes. self.assert_lint('string Class<Type>::Method() const\n' @@ -2028,21 +2046,27 @@ class CppStyleTest(CppStyleTestBase): self.assert_lint(' char* oneSpaceIndent = "public:";', 'Weird number of spaces at line-start. ' 'Are you using a 4-space indent? [whitespace/indent] [3]') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - - def test_label(self): - self.assert_lint('public:', - 'Labels should always be indented at least one space. ' - 'If this is a member-initializer list in a constructor, ' - 'the colon should be on the line after the definition ' - 'header. [whitespace/labels] [4]') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_multi_line_lint( + 'class Foo {\n' + 'public:\n' + ' enum Bar {\n' + ' Alpha,\n' + ' Beta,\n' + '#if ENABLED_BETZ\n' + ' Charlie,\n' + '#endif\n' + ' };\n' + '};', + '') def test_not_alabel(self): self.assert_lint('MyVeryLongNamespace::MyVeryLongClassName::', '') @@ -2080,8 +2104,9 @@ class CppStyleTest(CppStyleTestBase): 'class Foo;', '') self.assert_multi_line_lint( - '''struct Foo* - foo = NewFoo();''', + '''\ + struct Foo* + foo = NewFoo();''', '') # Here is an example where the linter gets confused, even though # the code doesn't violate the style guide. @@ -3171,31 +3196,35 @@ class NoNonVirtualDestructorsTest(CppStyleTestBase): def test_no_error(self): self.assert_multi_line_lint( - '''class Foo { - virtual ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + virtual ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo { - virtual inline ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + virtual inline ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo { - inline virtual ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + inline virtual ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo::Goo { - virtual ~Goo(); - virtual void goo(); - };''', + '''\ + class Foo::Goo { + virtual ~Goo(); + virtual void goo(); + };''', '') self.assert_multi_line_lint( 'class Foo { void foo(); };', @@ -3215,92 +3244,92 @@ class NoNonVirtualDestructorsTest(CppStyleTestBase): 'More than one command on the same line [whitespace/newline] [4]') self.assert_multi_line_lint( - '''class Qualified::Goo : public Foo { - virtual void goo(); - };''', - '') - - self.assert_multi_line_lint( - # Line-ending : - '''class Goo : - public Foo { + '''\ + class Qualified::Goo : public Foo { virtual void goo(); - };''', - 'Labels should always be indented at least one space. If this is a ' - 'member-initializer list in a constructor, the colon should be on the ' - 'line after the definition header. [whitespace/labels] [4]') + };''', + '') def test_no_destructor_when_virtual_needed(self): self.assert_multi_line_lint_re( - '''class Foo { - virtual void foo(); - };''', + '''\ + class Foo { + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_destructor_non_virtual_when_virtual_needed(self): self.assert_multi_line_lint_re( - '''class Foo { - ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + ~Foo(); + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_no_warn_when_derived(self): self.assert_multi_line_lint( - '''class Foo : public Goo { - virtual void foo(); - };''', + '''\ + class Foo : public Goo { + virtual void foo(); + };''', '') def test_internal_braces(self): self.assert_multi_line_lint_re( - '''class Foo { - enum Goo { - GOO - }; - virtual void foo(); - };''', + '''\ + class Foo { + enum Goo { + GOO + }; + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_inner_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Foo { - class Goo { - virtual void goo(); - }; - };''', + '''\ + class Foo { + class Goo { + virtual void goo(); + }; + };''', 'The class Goo probably needs a virtual destructor') def test_outer_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Foo { - class Goo { - }; - virtual void foo(); - };''', + '''\ + class Foo { + class Goo { + }; + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_qualified_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Qualified::Foo { - virtual void foo(); - };''', + '''\ + class Qualified::Foo { + virtual void foo(); + };''', 'The class Qualified::Foo probably needs a virtual destructor') def test_multi_line_declaration_no_error(self): self.assert_multi_line_lint_re( - '''class Foo - : public Goo { - virtual void foo(); - };''', + '''\ + class Foo + : public Goo { + virtual void foo(); + };''', '') def test_multi_line_declaration_with_error(self): self.assert_multi_line_lint( - '''class Foo - { - virtual void foo(); - };''', + '''\ + class Foo + { + virtual void foo(); + };''', ['This { should be at the end of the previous line ' '[whitespace/braces] [4]', 'The class Foo probably needs a virtual destructor due to having ' @@ -3495,7 +3524,6 @@ class WebKitStyleTest(CppStyleTestBase): ' int goo;\n' '};', 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') - # FIXME: No tests for 8-spaces. # 3. In a header, code inside a namespace should not be indented. self.assert_multi_line_lint( @@ -4180,7 +4208,14 @@ class WebKitStyleTest(CppStyleTestBase): ' myFunction(reallyLongParam1, reallyLongParam2,\n' ' reallyLongParam3);\n' '}\n', - '') + 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') + + self.assert_multi_line_lint( + 'if (true) {\n' + ' myFunction(reallyLongParam1, reallyLongParam2,\n' + ' reallyLongParam3);\n' + '}\n', + 'When wrapping a line, only indent 4 spaces. [whitespace/indent] [3]') # 4. Control clauses without a body should use empty braces. self.assert_multi_line_lint( @@ -4189,7 +4224,7 @@ class WebKitStyleTest(CppStyleTestBase): self.assert_multi_line_lint( 'for ( ; current;\n' ' current = current->next) { }\n', - '') + 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') self.assert_multi_line_lint( 'for ( ; current; current = current->next);\n', 'Semicolon defining empty statement for this loop. Use { } instead. [whitespace/semicolon] [5]') diff --git a/Tools/Scripts/webkitpy/style/checkers/test_expectations.py b/Tools/Scripts/webkitpy/style/checkers/test_expectations.py index 46403b7db..1ce40cd39 100644 --- a/Tools/Scripts/webkitpy/style/checkers/test_expectations.py +++ b/Tools/Scripts/webkitpy/style/checkers/test_expectations.py @@ -67,7 +67,7 @@ class TestExpectationsChecker(object): # FIXME: host should be a required parameter, not an optional one. host = host or Host() - host._initialize_scm() + host.initialize_scm() self._port_obj = self._determine_port_from_expectations_path(host, file_path) @@ -94,8 +94,6 @@ class TestExpectationsChecker(object): expectations = '\n'.join(lines) if self._port_obj: self.check_test_expectations(expectations_str=expectations, tests=None) - else: - self._handle_style_error(1, 'test/expectations', 5, - 'No port uses path %s for test_expectations' % self._file_path) + # Warn tabs in lines as well self.check_tabs(lines) diff --git a/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py b/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py index f12397787..1516de797 100644 --- a/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py @@ -82,6 +82,10 @@ class TestExpectationsTestCase(unittest.TestCase): self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl/TestExpectations') self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl-wk1/TestExpectations') self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl-wk2/TestExpectations') + self._expect_port_for_expectations_path('qt', 'LayoutTests/platform/qt-win/TestExpectations') + # FIXME: check-webkit-style doesn't know how to create port objects for all Qt version (4.8, 5.0) and + # will only check files based on the installed version of Qt. + #self._expect_port_for_expectations_path('qt', 'LayoutTests/platform/qt-5.0-wk2/TestExpectations') def assert_lines_lint(self, lines, should_pass, expected_output=None): self._error_collector.reset_errors() diff --git a/Tools/Scripts/webkitpy/style/main.py b/Tools/Scripts/webkitpy/style/main.py index e90d98a42..574368a3e 100644 --- a/Tools/Scripts/webkitpy/style/main.py +++ b/Tools/Scripts/webkitpy/style/main.py @@ -124,7 +124,7 @@ class CheckWebKitStyle(object): args = sys.argv[1:] host = Host() - host._initialize_scm() + host.initialize_scm() stderr = self._engage_awesome_stderr_hacks() diff --git a/Tools/Scripts/webkitpy/test/main.py b/Tools/Scripts/webkitpy/test/main.py index 28de8a6e0..852413299 100644 --- a/Tools/Scripts/webkitpy/test/main.py +++ b/Tools/Scripts/webkitpy/test/main.py @@ -81,7 +81,7 @@ class Tester(object): def skip(self, names, reason, bugid): self.finder.skip(names, reason, bugid) - def _parse_args(self): + def _parse_args(self, argv=None): parser = optparse.OptionParser(usage='usage: %prog [options] [args...]') parser.add_option('-a', '--all', action='store_true', default=False, help='run all the tests') @@ -103,7 +103,7 @@ class Tester(object): parser.epilog = ('[args...] is an optional list of modules, test_classes, or individual tests. ' 'If no args are given, all the tests will be run.') - return parser.parse_args() + return parser.parse_args(argv) def run(self): self._options, args = self._parse_args() @@ -188,10 +188,15 @@ class Tester(object): loader.test_method_prefixes = [] serial_tests = [] - loader.test_method_prefixes.extend(['serial_test_', 'serial_integration_test_']) + loader.test_method_prefixes = ['serial_test_', 'serial_integration_test_'] for name in names: serial_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) + # loader.loadTestsFromName() will not verify that names begin with one of the test_method_prefixes + # if the names were explicitly provided (e.g., MainTest.test_basic), so this means that any individual + # tests will be included in both parallel_tests and serial_tests, and we need to de-dup them. + serial_tests = list(set(serial_tests).difference(set(parallel_tests))) + return (parallel_tests, serial_tests) def _all_test_names(self, suite): diff --git a/Tools/Scripts/webkitpy/test/main_unittest.py b/Tools/Scripts/webkitpy/test/main_unittest.py index ca7ebba0e..2020f5b60 100644 --- a/Tools/Scripts/webkitpy/test/main_unittest.py +++ b/Tools/Scripts/webkitpy/test/main_unittest.py @@ -25,7 +25,7 @@ import unittest import StringIO from webkitpy.common.system.outputcapture import OutputCapture -from webkitpy.test.main import Tester +from webkitpy.test.main import Tester, _Loader class TesterTest(unittest.TestCase): @@ -52,3 +52,10 @@ class TesterTest(unittest.TestCase): self.assertTrue('No tests to run' in errors.getvalue()) self.assertTrue('No tests to run' in logs) + + def test_individual_names_are_not_run_twice(self): + tester = Tester() + tester._options, args = tester._parse_args(["webkitpy.test.main_unittest.TesterTest.test_no_tests_found"]) + parallel_tests, serial_tests = tester._test_names(_Loader(), args) + self.assertEquals(parallel_tests, args) + self.assertEquals(serial_tests, []) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py index c154da4a1..454ae0c45 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py @@ -34,7 +34,8 @@ mod_pywebsocket is a WebSocket extension for Apache HTTP Server intended for testing or experimental purposes. mod_python is required. -Installation: +Installation +============ 0. Prepare an Apache HTTP Server for which mod_python is enabled. @@ -60,11 +61,6 @@ Installation: <scan_dir> is useful in saving scan time when <websock_handlers> contains many non-WebSocket handler files. - If you want to support old handshake based on - draft-hixie-thewebsocketprotocol-75: - - PythonOption mod_pywebsocket.allow_draft75 On - If you want to allow handlers whose canonical path is not under the root directory (i.e. symbolic link is in root directory but its target is not), configure as follows: @@ -89,7 +85,8 @@ Installation: 3. Verify installation. You can use example/console.html to poke the server. -Writing WebSocket handlers: +Writing WebSocket handlers +========================== When a WebSocket request comes in, the resource name specified in the handshake is considered as if it is a file path under @@ -118,28 +115,36 @@ extra handshake (web_socket_do_extra_handshake): - ws_resource - ws_origin - ws_version -- ws_location (Hixie 75 and HyBi 00 only) -- ws_extensions (Hybi 06 and later) +- ws_location (HyBi 00 only) +- ws_extensions (HyBi 06 and later) - ws_deflate (HyBi 06 and later) - ws_protocol - ws_requested_protocols (HyBi 06 and later) -The last two are a bit tricky. +The last two are a bit tricky. See the next subsection. + + +Subprotocol Negotiation +----------------------- For HyBi 06 and later, ws_protocol is always set to None when web_socket_do_extra_handshake is called. If ws_requested_protocols is not None, you must choose one subprotocol from this list and set it to ws_protocol. -For Hixie 75 and HyBi 00, when web_socket_do_extra_handshake is called, +For HyBi 00, when web_socket_do_extra_handshake is called, ws_protocol is set to the value given by the client in -Sec-WebSocket-Protocol (WebSocket-Protocol for Hixie 75) header or None if +Sec-WebSocket-Protocol header or None if such header was not found in the opening handshake request. Finish extra handshake with ws_protocol untouched to accept the request subprotocol. -Then, Sec-WebSocket-Protocol (or WebSocket-Protocol) header will be sent to +Then, Sec-WebSocket-Protocol header will be sent to the client in response with the same value as requested. Raise an exception in web_socket_do_extra_handshake to reject the requested subprotocol. + +Data Transfer +------------- + web_socket_transfer_data is called after the handshake completed successfully. A handler can receive/send messages from/to the client using request. mod_pywebsocket.msgutil module provides utilities @@ -159,12 +164,16 @@ You can send a message by the following statement. request.ws_stream.send_message(message) + +Closing Connection +------------------ + Executing the following statement or just return-ing from web_socket_transfer_data cause connection close. request.ws_stream.close_connection() -When you're using IETF HyBi 00 or later protocol, close_connection will wait +close_connection will wait for closing handshake acknowledgement coming from the client. When it couldn't receive a valid acknowledgement, raises an exception. @@ -176,6 +185,10 @@ use in web_socket_passive_closing_handshake. - ws_close_code - ws_close_reason + +Threading +--------- + A WebSocket handler must be thread-safe if the server (Apache or standalone.py) is configured to use threads. """ diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py index c84ca6e07..94cf5b31b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py @@ -32,7 +32,8 @@ protocol version HyBi 00 and Hixie 75. Specification: -http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +- HyBi 00 http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +- Hixie 75 http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 """ diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py index 34fa7a60e..bd158fa6b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py @@ -37,6 +37,7 @@ http://tools.ietf.org/html/rfc6455 from collections import deque +import logging import os import struct import time @@ -162,14 +163,145 @@ def create_text_frame( frame_filters) +def parse_frame(receive_bytes, logger=None, + ws_version=common.VERSION_HYBI_LATEST, + unmask_receive=True): + """Parses a frame. Returns a tuple containing each header field and + payload. + + Args: + receive_bytes: a function that reads frame data from a stream or + something similar. The function takes length of the bytes to be + read. The function must raise ConnectionTerminatedException if + there is not enough data to be read. + logger: a logging object. + ws_version: the version of WebSocket protocol. + unmask_receive: unmask received frames. When received unmasked + frame, raises InvalidFrameException. + + Raises: + ConnectionTerminatedException: when receive_bytes raises it. + InvalidFrameException: when the frame contains invalid data. + """ + + if not logger: + logger = logging.getLogger() + + logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') + + received = receive_bytes(2) + + first_byte = ord(received[0]) + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xf + + second_byte = ord(received[1]) + mask = (second_byte >> 7) & 1 + payload_length = second_byte & 0x7f + + logger.log(common.LOGLEVEL_FINE, + 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' + 'Mask=%s, Payload_length=%s', + fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) + + if (mask == 1) != unmask_receive: + raise InvalidFrameException( + 'Mask bit on the received frame did\'nt match masking ' + 'configuration for received frames') + + # The HyBi and later specs disallow putting a value in 0x0-0xFFFF + # into the 8-octet extended payload length field (or 0x0-0xFD in + # 2-octet field). + valid_length_encoding = True + length_encoding_bytes = 1 + if payload_length == 127: + logger.log(common.LOGLEVEL_FINE, + 'Receive 8-octet extended payload length') + + extended_payload_length = receive_bytes(8) + payload_length = struct.unpack( + '!Q', extended_payload_length)[0] + if payload_length > 0x7FFFFFFFFFFFFFFF: + raise InvalidFrameException( + 'Extended payload length >= 2^63') + if ws_version >= 13 and payload_length < 0x10000: + valid_length_encoding = False + length_encoding_bytes = 8 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + elif payload_length == 126: + logger.log(common.LOGLEVEL_FINE, + 'Receive 2-octet extended payload length') + + extended_payload_length = receive_bytes(2) + payload_length = struct.unpack( + '!H', extended_payload_length)[0] + if ws_version >= 13 and payload_length < 126: + valid_length_encoding = False + length_encoding_bytes = 2 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + + if not valid_length_encoding: + logger.warning( + 'Payload length is not encoded using the minimal number of ' + 'bytes (%d is encoded using %d bytes)', + payload_length, + length_encoding_bytes) + + if mask == 1: + logger.log(common.LOGLEVEL_FINE, 'Receive mask') + + masking_nonce = receive_bytes(4) + masker = util.RepeatedXorMasker(masking_nonce) + + logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) + else: + masker = _NOOP_MASKER + + logger.log(common.LOGLEVEL_FINE, 'Receive payload data') + if logger.isEnabledFor(common.LOGLEVEL_FINE): + receive_start = time.time() + + raw_payload_bytes = receive_bytes(payload_length) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done receiving payload data at %s MB/s', + payload_length / (time.time() - receive_start) / 1000 / 1000) + logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + unmask_start = time.time() + + bytes = masker.mask(raw_payload_bytes) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done unmasking payload data at %s MB/s', + payload_length / (time.time() - unmask_start) / 1000 / 1000) + + return opcode, bytes, fin, rsv1, rsv2, rsv3 + + class FragmentedFrameBuilder(object): """A stateful class to send a message as fragments.""" - def __init__(self, mask, frame_filters=[]): + def __init__(self, mask, frame_filters=[], encode_utf8=True): """Constructs an instance.""" self._mask = mask self._frame_filters = frame_filters + # This is for skipping UTF-8 encoding when building text type frames + # from compressed data. + self._encode_utf8 = encode_utf8 self._started = False @@ -177,7 +309,7 @@ class FragmentedFrameBuilder(object): # frames in the message are all the same. self._opcode = common.OPCODE_TEXT - def build(self, message, end, binary): + def build(self, payload_data, end, binary): if binary: frame_type = common.OPCODE_BINARY else: @@ -198,12 +330,12 @@ class FragmentedFrameBuilder(object): self._started = True fin = 0 - if binary: + if binary or not self._encode_utf8: return create_binary_frame( - message, opcode, fin, self._mask, self._frame_filters) + payload_data, opcode, fin, self._mask, self._frame_filters) else: return create_text_frame( - message, opcode, fin, self._mask, self._frame_filters) + payload_data, opcode, fin, self._mask, self._frame_filters) def _create_control_frame(opcode, body, mask, frame_filters): @@ -235,6 +367,22 @@ def create_close_frame(body, mask=False, frame_filters=[]): common.OPCODE_CLOSE, body, mask, frame_filters) +def create_closing_handshake_body(code, reason): + body = '' + if code is not None: + if (code > common.STATUS_USER_PRIVATE_MAX or + code < common.STATUS_NORMAL_CLOSURE): + raise BadOperationException('Status code is out of range') + if (code == common.STATUS_NO_STATUS_RECEIVED or + code == common.STATUS_ABNORMAL_CLOSURE or + code == common.STATUS_TLS_HANDSHAKE): + raise BadOperationException('Status code is reserved pseudo ' + 'code') + encoded_reason = reason.encode('utf-8') + body = struct.pack('!H', code) + encoded_reason + return body + + class StreamOptions(object): """Holds option values to configure Stream objects.""" @@ -248,8 +396,16 @@ class StreamOptions(object): self.outgoing_frame_filters = [] self.incoming_frame_filters = [] + # Filters applied to messages. Control frames are not affected by them. + self.outgoing_message_filters = [] + self.incoming_message_filters = [] + + self.encode_text_message_to_utf8 = True self.mask_send = False self.unmask_receive = True + # RFC6455 disallows fragmented control frames, but mux extension + # relaxes the restriction. + self.allow_fragmented_control_frame = False class Stream(StreamBase): @@ -283,7 +439,8 @@ class Stream(StreamBase): self._original_opcode = None self._writer = FragmentedFrameBuilder( - self._options.mask_send, self._options.outgoing_frame_filters) + self._options.mask_send, self._options.outgoing_frame_filters, + self._options.encode_text_message_to_utf8) self._ping_queue = deque() @@ -297,109 +454,13 @@ class Stream(StreamBase): InvalidFrameException: when the frame contains invalid data. """ - self._logger.log(common.LOGLEVEL_FINE, - 'Receive the first 2 octets of a frame') - - received = self.receive_bytes(2) - - first_byte = ord(received[0]) - fin = (first_byte >> 7) & 1 - rsv1 = (first_byte >> 6) & 1 - rsv2 = (first_byte >> 5) & 1 - rsv3 = (first_byte >> 4) & 1 - opcode = first_byte & 0xf - - second_byte = ord(received[1]) - mask = (second_byte >> 7) & 1 - payload_length = second_byte & 0x7f - - self._logger.log(common.LOGLEVEL_FINE, - 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' - 'Mask=%s, Payload_length=%s', - fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) - - if (mask == 1) != self._options.unmask_receive: - raise InvalidFrameException( - 'Mask bit on the received frame did\'nt match masking ' - 'configuration for received frames') - - # The Hybi-13 and later specs disallow putting a value in 0x0-0xFFFF - # into the 8-octet extended payload length field (or 0x0-0xFD in - # 2-octet field). - valid_length_encoding = True - length_encoding_bytes = 1 - if payload_length == 127: - self._logger.log(common.LOGLEVEL_FINE, - 'Receive 8-octet extended payload length') - - extended_payload_length = self.receive_bytes(8) - payload_length = struct.unpack( - '!Q', extended_payload_length)[0] - if payload_length > 0x7FFFFFFFFFFFFFFF: - raise InvalidFrameException( - 'Extended payload length >= 2^63') - if self._request.ws_version >= 13 and payload_length < 0x10000: - valid_length_encoding = False - length_encoding_bytes = 8 - - self._logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - elif payload_length == 126: - self._logger.log(common.LOGLEVEL_FINE, - 'Receive 2-octet extended payload length') - - extended_payload_length = self.receive_bytes(2) - payload_length = struct.unpack( - '!H', extended_payload_length)[0] - if self._request.ws_version >= 13 and payload_length < 126: - valid_length_encoding = False - length_encoding_bytes = 2 - - self._logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - - if not valid_length_encoding: - self._logger.warning( - 'Payload length is not encoded using the minimal number of ' - 'bytes (%d is encoded using %d bytes)', - payload_length, - length_encoding_bytes) - - if mask == 1: - self._logger.log(common.LOGLEVEL_FINE, 'Receive mask') - - masking_nonce = self.receive_bytes(4) - masker = util.RepeatedXorMasker(masking_nonce) - - self._logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) - else: - masker = _NOOP_MASKER - - self._logger.log(common.LOGLEVEL_FINE, 'Receive payload data') - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - receive_start = time.time() - - raw_payload_bytes = self.receive_bytes(payload_length) - - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - self._logger.log( - common.LOGLEVEL_FINE, - 'Done receiving payload data at %s MB/s', - payload_length / (time.time() - receive_start) / 1000 / 1000) - self._logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') - - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - unmask_start = time.time() - - bytes = masker.mask(raw_payload_bytes) + def _receive_bytes(length): + return self.receive_bytes(length) - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - self._logger.log( - common.LOGLEVEL_FINE, - 'Done unmasking payload data at %s MB/s', - payload_length / (time.time() - unmask_start) / 1000 / 1000) - - return opcode, bytes, fin, rsv1, rsv2, rsv3 + return parse_frame(receive_bytes=_receive_bytes, + logger=self._logger, + ws_version=self._request.ws_version, + unmask_receive=self._options.unmask_receive) def _receive_frame_as_frame_object(self): opcode, bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() @@ -407,6 +468,32 @@ class Stream(StreamBase): return Frame(fin=fin, rsv1=rsv1, rsv2=rsv2, rsv3=rsv3, opcode=opcode, payload=bytes) + def receive_filtered_frame(self): + """Receives a frame and applies frame filters and message filters. + The frame to be received must satisfy following conditions: + - The frame is not fragmented. + - The opcode of the frame is TEXT or BINARY. + + DO NOT USE this method except for testing purpose. + """ + + frame = self._receive_frame_as_frame_object() + if not frame.fin: + raise InvalidFrameException( + 'Segmented frames must not be received via ' + 'receive_filtered_frame()') + if (frame.opcode != common.OPCODE_TEXT and + frame.opcode != common.OPCODE_BINARY): + raise InvalidFrameException( + 'Control frames must not be received via ' + 'receive_filtered_frame()') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + for message_filter in self._options.incoming_message_filters: + frame.payload = message_filter.filter(frame.payload) + return frame + def send_message(self, message, end=True, binary=False): """Send message. @@ -428,11 +515,219 @@ class Stream(StreamBase): raise BadOperationException( 'Message for binary frame must be instance of str') + for message_filter in self._options.outgoing_message_filters: + message = message_filter.filter(message, end, binary) + try: - self._write(self._writer.build(message, end, binary)) + # Set this to any positive integer to limit maximum size of data in + # payload data of each frame. + MAX_PAYLOAD_DATA_SIZE = -1 + + if MAX_PAYLOAD_DATA_SIZE <= 0: + self._write(self._writer.build(message, end, binary)) + return + + bytes_written = 0 + while True: + end_for_this_frame = end + bytes_to_write = len(message) - bytes_written + if (MAX_PAYLOAD_DATA_SIZE > 0 and + bytes_to_write > MAX_PAYLOAD_DATA_SIZE): + end_for_this_frame = False + bytes_to_write = MAX_PAYLOAD_DATA_SIZE + + frame = self._writer.build( + message[bytes_written:bytes_written + bytes_to_write], + end_for_this_frame, + binary) + self._write(frame) + + bytes_written += bytes_to_write + + # This if must be placed here (the end of while block) so that + # at least one frame is sent. + if len(message) <= bytes_written: + break except ValueError, e: raise BadOperationException(e) + def _get_message_from_frame(self, frame): + """Gets a message from frame. If the message is composed of fragmented + frames and the frame is not the last fragmented frame, this method + returns None. The whole message will be returned when the last + fragmented frame is passed to this method. + + Raises: + InvalidFrameException: when the frame doesn't match defragmentation + context, or the frame contains invalid data. + """ + + if frame.opcode == common.OPCODE_CONTINUATION: + if not self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received a termination frame but fragmentation ' + 'not started') + else: + raise InvalidFrameException( + 'Received an intermediate frame but ' + 'fragmentation not started') + + if frame.fin: + # End of fragmentation frame + self._received_fragments.append(frame.payload) + message = ''.join(self._received_fragments) + self._received_fragments = [] + return message + else: + # Intermediate frame + self._received_fragments.append(frame.payload) + return None + else: + if self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received an unfragmented frame without ' + 'terminating existing fragmentation') + else: + raise InvalidFrameException( + 'New fragmentation started without terminating ' + 'existing fragmentation') + + if frame.fin: + # Unfragmented frame + + self._original_opcode = frame.opcode + return frame.payload + else: + # Start of fragmentation frame + + if (not self._options.allow_fragmented_control_frame and + common.is_control_opcode(frame.opcode)): + raise InvalidFrameException( + 'Control frames must not be fragmented') + + self._original_opcode = frame.opcode + self._received_fragments.append(frame.payload) + return None + + def _process_close_message(self, message): + """Processes close message. + + Args: + message: close message. + + Raises: + InvalidFrameException: when the message is invalid. + """ + + self._request.client_terminated = True + + # Status code is optional. We can have status reason only if we + # have status code. Status reason can be empty string. So, + # allowed cases are + # - no application data: no code no reason + # - 2 octet of application data: has code but no reason + # - 3 or more octet of application data: both code and reason + if len(message) == 0: + self._logger.debug('Received close frame (empty body)') + self._request.ws_close_code = ( + common.STATUS_NO_STATUS_RECEIVED) + elif len(message) == 1: + raise InvalidFrameException( + 'If a close frame has status code, the length of ' + 'status code must be 2 octet') + elif len(message) >= 2: + self._request.ws_close_code = struct.unpack( + '!H', message[0:2])[0] + self._request.ws_close_reason = message[2:].decode( + 'utf-8', 'replace') + self._logger.debug( + 'Received close frame (code=%d, reason=%r)', + self._request.ws_close_code, + self._request.ws_close_reason) + + # Drain junk data after the close frame if necessary. + self._drain_received_data() + + if self._request.server_terminated: + self._logger.debug( + 'Received ack for server-initiated closing handshake') + return + + self._logger.debug( + 'Received client-initiated closing handshake') + + code = common.STATUS_NORMAL_CLOSURE + reason = '' + if hasattr(self._request, '_dispatcher'): + dispatcher = self._request._dispatcher + code, reason = dispatcher.passive_closing_handshake( + self._request) + if code is None and reason is not None and len(reason) > 0: + self._logger.warning( + 'Handler specified reason despite code being None') + reason = '' + if reason is None: + reason = '' + self._send_closing_handshake(code, reason) + self._logger.debug( + 'Sent ack for client-initiated closing handshake ' + '(code=%r, reason=%r)', code, reason) + + def _process_ping_message(self, message): + """Processes ping message. + + Args: + message: ping message. + """ + + try: + handler = self._request.on_ping_handler + if handler: + handler(self._request, message) + return + except AttributeError, e: + pass + self._send_pong(message) + + def _process_pong_message(self, message): + """Processes pong message. + + Args: + message: pong message. + """ + + # TODO(tyoshino): Add ping timeout handling. + + inflight_pings = deque() + + while True: + try: + expected_body = self._ping_queue.popleft() + if expected_body == message: + # inflight_pings contains pings ignored by the + # other peer. Just forget them. + self._logger.debug( + 'Ping %r is acked (%d pings were ignored)', + expected_body, len(inflight_pings)) + break + else: + inflight_pings.append(expected_body) + except IndexError, e: + # The received pong was unsolicited pong. Keep the + # ping queue as is. + self._ping_queue = inflight_pings + self._logger.debug('Received a unsolicited pong') + break + + try: + handler = self._request.on_pong_handler + if handler: + handler(self._request, message) + except AttributeError, e: + pass + def receive_message(self): """Receive a WebSocket frame and return its payload as a text in unicode or a binary in str. @@ -482,52 +777,12 @@ class Stream(StreamBase): 'Unsupported flag is set (rsv = %d%d%d)' % (frame.rsv1, frame.rsv2, frame.rsv3)) - if frame.opcode == common.OPCODE_CONTINUATION: - if not self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received a termination frame but fragmentation ' - 'not started') - else: - raise InvalidFrameException( - 'Received an intermediate frame but ' - 'fragmentation not started') - - if frame.fin: - # End of fragmentation frame - self._received_fragments.append(frame.payload) - message = ''.join(self._received_fragments) - self._received_fragments = [] - else: - # Intermediate frame - self._received_fragments.append(frame.payload) - continue - else: - if self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received an unfragmented frame without ' - 'terminating existing fragmentation') - else: - raise InvalidFrameException( - 'New fragmentation started without terminating ' - 'existing fragmentation') - - if frame.fin: - # Unfragmented frame - - self._original_opcode = frame.opcode - message = frame.payload - else: - # Start of fragmentation frame - - if common.is_control_opcode(frame.opcode): - raise InvalidFrameException( - 'Control frames must not be fragmented') + message = self._get_message_from_frame(frame) + if message is None: + continue - self._original_opcode = frame.opcode - self._received_fragments.append(frame.payload) - continue + for message_filter in self._options.incoming_message_filters: + message = message_filter.filter(message) if self._original_opcode == common.OPCODE_TEXT: # The WebSocket protocol section 4.4 specifies that invalid @@ -540,124 +795,21 @@ class Stream(StreamBase): elif self._original_opcode == common.OPCODE_BINARY: return message elif self._original_opcode == common.OPCODE_CLOSE: - self._request.client_terminated = True - - # Status code is optional. We can have status reason only if we - # have status code. Status reason can be empty string. So, - # allowed cases are - # - no application data: no code no reason - # - 2 octet of application data: has code but no reason - # - 3 or more octet of application data: both code and reason - if len(message) == 0: - self._logger.debug('Received close frame (empty body)') - self._request.ws_close_code = ( - common.STATUS_NO_STATUS_RECEIVED) - elif len(message) == 1: - raise InvalidFrameException( - 'If a close frame has status code, the length of ' - 'status code must be 2 octet') - elif len(message) >= 2: - self._request.ws_close_code = struct.unpack( - '!H', message[0:2])[0] - self._request.ws_close_reason = message[2:].decode( - 'utf-8', 'replace') - self._logger.debug( - 'Received close frame (code=%d, reason=%r)', - self._request.ws_close_code, - self._request.ws_close_reason) - - # Drain junk data after the close frame if necessary. - self._drain_received_data() - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing handshake') - return None - - self._logger.debug( - 'Received client-initiated closing handshake') - - code = common.STATUS_NORMAL_CLOSURE - reason = '' - if hasattr(self._request, '_dispatcher'): - dispatcher = self._request._dispatcher - code, reason = dispatcher.passive_closing_handshake( - self._request) - if code is None and reason is not None and len(reason) > 0: - self._logger.warning( - 'Handler specified reason despite code being None') - reason = '' - if reason is None: - reason = '' - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent ack for client-initiated closing handshake ' - '(code=%r, reason=%r)', code, reason) + self._process_close_message(message) return None elif self._original_opcode == common.OPCODE_PING: - try: - handler = self._request.on_ping_handler - if handler: - handler(self._request, message) - continue - except AttributeError, e: - pass - self._send_pong(message) + self._process_ping_message(message) elif self._original_opcode == common.OPCODE_PONG: - # TODO(tyoshino): Add ping timeout handling. - - inflight_pings = deque() - - while True: - try: - expected_body = self._ping_queue.popleft() - if expected_body == message: - # inflight_pings contains pings ignored by the - # other peer. Just forget them. - self._logger.debug( - 'Ping %r is acked (%d pings were ignored)', - expected_body, len(inflight_pings)) - break - else: - inflight_pings.append(expected_body) - except IndexError, e: - # The received pong was unsolicited pong. Keep the - # ping queue as is. - self._ping_queue = inflight_pings - self._logger.debug('Received a unsolicited pong') - break - - try: - handler = self._request.on_pong_handler - if handler: - handler(self._request, message) - continue - except AttributeError, e: - pass - - continue + self._process_pong_message(message) else: raise UnsupportedFrameException( 'Opcode %d is not supported' % self._original_opcode) def _send_closing_handshake(self, code, reason): - body = '' - if code is not None: - if (code > common.STATUS_USER_PRIVATE_MAX or - code < common.STATUS_NORMAL_CLOSURE): - raise BadOperationException('Status code is out of range') - if (code == common.STATUS_NO_STATUS_RECEIVED or - code == common.STATUS_ABNORMAL_CLOSURE or - code == common.STATUS_TLS_HANDSHAKE): - raise BadOperationException('Status code is reserved pseudo ' - 'code') - encoded_reason = reason.encode('utf-8') - body = struct.pack('!H', code) + encoded_reason - + body = create_closing_handshake_body(code, reason) frame = create_close_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) + body, mask=self._options.mask_send, + frame_filters=self._options.outgoing_frame_filters) self._request.server_terminated = True @@ -731,6 +883,14 @@ class Stream(StreamBase): self._options.outgoing_frame_filters) self._write(frame) + def get_last_received_opcode(self): + """Returns the opcode of the WebSocket message which the last received + frame belongs to. The return value is valid iff immediately after + receive_message call. + """ + + return self._original_opcode + def _drain_received_data(self): """Drains unread data in the receive buffer to avoid sending out TCP RST packet. This is because when deflate-stream is enabled, some diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py index 710967c80..2388379c0 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py @@ -104,7 +104,10 @@ SEC_WEBSOCKET_LOCATION_HEADER = 'Sec-WebSocket-Location' DEFLATE_STREAM_EXTENSION = 'deflate-stream' DEFLATE_FRAME_EXTENSION = 'deflate-frame' PERFRAME_COMPRESSION_EXTENSION = 'perframe-compress' +PERMESSAGE_COMPRESSION_EXTENSION = 'permessage-compress' X_WEBKIT_DEFLATE_FRAME_EXTENSION = 'x-webkit-deflate-frame' +X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION = 'x-webkit-permessage-compress' +MUX_EXTENSION = 'mux_DO_NOT_USE' # Status codes # Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and @@ -125,7 +128,7 @@ STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 STATUS_POLICY_VIOLATION = 1008 STATUS_MESSAGE_TOO_BIG = 1009 STATUS_MANDATORY_EXTENSION = 1010 -STATUS_INTERNAL_SERVER_ERROR = 1011 +STATUS_INTERNAL_ENDPOINT_ERROR = 1011 STATUS_TLS_HANDSHAKE = 1015 STATUS_USER_REGISTERED_BASE = 3000 STATUS_USER_REGISTERED_MAX = 3999 diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py index ab1eb4fb3..25905f180 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py @@ -39,6 +39,7 @@ import re from mod_pywebsocket import common from mod_pywebsocket import handshake from mod_pywebsocket import msgutil +from mod_pywebsocket import mux from mod_pywebsocket import stream from mod_pywebsocket import util @@ -277,13 +278,18 @@ class Dispatcher(object): AbortedByUserException: when user handler abort connection """ - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % request.ws_resource) - transfer_data_ = handler_suite.transfer_data # TODO(tyoshino): Terminate underlying TCP connection if possible. try: - transfer_data_(request) + if mux.use_mux(request): + mux.start(request, self) + else: + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % + request.ws_resource) + transfer_data_ = handler_suite.transfer_data + transfer_data_(request) + if not request.server_terminated: request.ws_stream.close_connection() # Catch non-critical exceptions the handler didn't handle. diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py index 52b7a4a19..c3ccc46a8 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py @@ -38,6 +38,9 @@ _available_processors = {} class ExtensionProcessorInterface(object): + def name(self): + return None + def get_extension_response(self): return None @@ -46,13 +49,21 @@ class ExtensionProcessorInterface(object): class DeflateStreamExtensionProcessor(ExtensionProcessorInterface): - """WebSocket DEFLATE stream extension processor.""" + """WebSocket DEFLATE stream extension processor. + + Specification: + Section 9.2.1 in + http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 + """ def __init__(self, request): self._logger = util.get_class_logger(self) self._request = request + def name(self): + return common.DEFLATE_STREAM_EXTENSION + def get_extension_response(self): if len(self._request.get_parameter_names()) != 0: return None @@ -70,8 +81,40 @@ _available_processors[common.DEFLATE_STREAM_EXTENSION] = ( DeflateStreamExtensionProcessor) +def _log_compression_ratio(logger, original_bytes, total_original_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if original_bytes != 0: + ratio = float(filtered_bytes) / original_bytes + if total_original_bytes != 0: + average_ratio = ( + float(total_filtered_bytes) / total_original_bytes) + logger.debug('Outgoing compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +def _log_decompression_ratio(logger, received_bytes, total_received_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if received_bytes != 0: + ratio = float(received_bytes) / filtered_bytes + if total_filtered_bytes != 0: + average_ratio = ( + float(total_received_bytes) / total_filtered_bytes) + logger.debug('Incoming compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame DEFLATE extension processor.""" + """WebSocket Per-frame DEFLATE extension processor. + + Specification: + http://tools.ietf.org/html/draft-tyoshino-hybi-websocket-perframe-deflate + """ _WINDOW_BITS_PARAM = 'max_window_bits' _NO_CONTEXT_TAKEOVER_PARAM = 'no_context_takeover' @@ -96,6 +139,9 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): # Total number of incoming bytes obtained after applying this filter. self._total_filtered_incoming_payload_bytes = 0 + def name(self): + return common.DEFLATE_FRAME_EXTENSION + def get_extension_response(self): # Any unknown parameter will be just ignored. @@ -199,18 +245,10 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): filtered_payload_size = len(frame.payload) self._total_filtered_outgoing_payload_bytes += filtered_payload_size - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if original_payload_size != 0: - ratio = float(filtered_payload_size) / original_payload_size - if self._total_outgoing_payload_bytes != 0: - average_ratio = ( - float(self._total_filtered_outgoing_payload_bytes) / - self._total_outgoing_payload_bytes) - self._logger.debug( - 'Outgoing compress ratio: %f (average: %f)' % - (ratio, average_ratio)) + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) def _incoming_filter(self, frame): """Transform incoming frames. This method is called only by @@ -231,18 +269,10 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): filtered_payload_size = len(frame.payload) self._total_filtered_incoming_payload_bytes += filtered_payload_size - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if received_payload_size != 0: - ratio = float(received_payload_size) / filtered_payload_size - if self._total_filtered_incoming_payload_bytes != 0: - average_ratio = ( - float(self._total_incoming_payload_bytes) / - self._total_filtered_incoming_payload_bytes) - self._logger.debug( - 'Incoming compress ratio: %f (average: %f)' % - (ratio, average_ratio)) + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) _available_processors[common.DEFLATE_FRAME_EXTENSION] = ( @@ -250,7 +280,7 @@ _available_processors[common.DEFLATE_FRAME_EXTENSION] = ( # Adding vendor-prefixed deflate-frame extension. -# TODO(bashi): Remove this after WebKit stops using vender prefix. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. _available_processors[common.X_WEBKIT_DEFLATE_FRAME_EXTENSION] = ( DeflateFrameExtensionProcessor) @@ -270,21 +300,22 @@ def _create_accepted_method_desc(method_name, method_params): return common.format_extension(extension) -class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame compression extension processor.""" +class CompressionExtensionProcessorBase(ExtensionProcessorInterface): + """Base class for Per-frame and Per-message compression extension.""" _METHOD_PARAM = 'method' - _DEFLATE_METHOD = 'deflate' def __init__(self, request): self._logger = util.get_class_logger(self) self._request = request self._compression_method_name = None self._compression_processor = None + self._compression_processor_hook = None + + def name(self): + return '' def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateFrameExtensionProcessor(method_desc) return None def _get_compression_processor_response(self): @@ -311,6 +342,10 @@ class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): break if compression_processor is None: return None + + if self._compression_processor_hook: + self._compression_processor_hook(compression_processor) + processor_response = compression_processor.get_extension_response() if processor_response is None: return None @@ -337,14 +372,340 @@ class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): return self._compression_processor.setup_stream_options(stream_options) + def set_compression_processor_hook(self, hook): + self._compression_processor_hook = hook + def get_compression_processor(self): return self._compression_processor +class PerFrameCompressionExtensionProcessor(CompressionExtensionProcessorBase): + """WebSocket Per-frame compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-websocket-perframe-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERFRAME_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateFrameExtensionProcessor(method_desc) + return None + + _available_processors[common.PERFRAME_COMPRESSION_EXTENSION] = ( PerFrameCompressionExtensionProcessor) +class DeflateMessageProcessor(ExtensionProcessorInterface): + """Per-message deflate processor.""" + + _S2C_MAX_WINDOW_BITS_PARAM = 's2c_max_window_bits' + _S2C_NO_CONTEXT_TAKEOVER_PARAM = 's2c_no_context_takeover' + _C2S_MAX_WINDOW_BITS_PARAM = 'c2s_max_window_bits' + _C2S_NO_CONTEXT_TAKEOVER_PARAM = 'c2s_no_context_takeover' + + def __init__(self, request): + self._request = request + self._logger = util.get_class_logger(self) + + self._c2s_max_window_bits = None + self._c2s_no_context_takeover = False + + self._compress_outgoing_enabled = False + + # True if a message is fragmented and compression is ongoing. + self._compress_ongoing = False + + # Counters for statistics. + + # Total number of outgoing bytes supplied to this filter. + self._total_outgoing_payload_bytes = 0 + # Total number of bytes sent to the network after applying this filter. + self._total_filtered_outgoing_payload_bytes = 0 + + # Total number of bytes received from the network. + self._total_incoming_payload_bytes = 0 + # Total number of incoming bytes obtained after applying this filter. + self._total_filtered_incoming_payload_bytes = 0 + + def name(self): + return 'deflate' + + def get_extension_response(self): + # Any unknown parameter will be just ignored. + + s2c_max_window_bits = self._request.get_parameter_value( + self._S2C_MAX_WINDOW_BITS_PARAM) + if s2c_max_window_bits is not None: + try: + s2c_max_window_bits = int(s2c_max_window_bits) + except ValueError, e: + return None + if s2c_max_window_bits < 8 or s2c_max_window_bits > 15: + return None + + s2c_no_context_takeover = self._request.has_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) + if (s2c_no_context_takeover and + self._request.get_parameter_value( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) is not None): + return None + + self._deflater = util._RFC1979Deflater( + s2c_max_window_bits, s2c_no_context_takeover) + + self._inflater = util._RFC1979Inflater() + + self._compress_outgoing_enabled = True + + response = common.ExtensionParameter(self._request.name()) + + if s2c_max_window_bits is not None: + response.add_parameter( + self._S2C_MAX_WINDOW_BITS_PARAM, str(s2c_max_window_bits)) + + if s2c_no_context_takeover: + response.add_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM, None) + + if self._c2s_max_window_bits is not None: + response.add_parameter( + self._C2S_MAX_WINDOW_BITS_PARAM, + str(self._c2s_max_window_bits)) + if self._c2s_no_context_takeover: + response.add_parameter( + self._C2S_NO_CONTEXT_TAKEOVER_PARAM, None) + + self._logger.debug( + 'Enable %s extension (' + 'request: s2c_max_window_bits=%s; s2c_no_context_takeover=%r, ' + 'response: c2s_max_window_bits=%s; c2s_no_context_takeover=%r)' % + (self._request.name(), + s2c_max_window_bits, + s2c_no_context_takeover, + self._c2s_max_window_bits, + self._c2s_no_context_takeover)) + + return response + + def setup_stream_options(self, stream_options): + class _OutgoingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, message, end=True, binary=False): + return self._parent._process_outgoing_message( + message, end, binary) + + class _IncomingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + self._decompress_next_message = False + + def decompress_next_message(self): + self._decompress_next_message = True + + def filter(self, message): + message = self._parent._process_incoming_message( + message, self._decompress_next_message) + self._decompress_next_message = False + return message + + self._outgoing_message_filter = _OutgoingMessageFilter(self) + self._incoming_message_filter = _IncomingMessageFilter(self) + stream_options.outgoing_message_filters.append( + self._outgoing_message_filter) + stream_options.incoming_message_filters.append( + self._incoming_message_filter) + + class _OutgoingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + self._set_compression_bit = False + + def set_compression_bit(self): + self._set_compression_bit = True + + def filter(self, frame): + self._parent._process_outgoing_frame( + frame, self._set_compression_bit) + self._set_compression_bit = False + + class _IncomingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._process_incoming_frame(frame) + + self._outgoing_frame_filter = _OutgoingFrameFilter(self) + self._incoming_frame_filter = _IncomingFrameFilter(self) + stream_options.outgoing_frame_filters.append( + self._outgoing_frame_filter) + stream_options.incoming_frame_filters.append( + self._incoming_frame_filter) + + stream_options.encode_text_message_to_utf8 = False + + def set_c2s_max_window_bits(self, value): + self._c2s_max_window_bits = value + + def set_c2s_no_context_takeover(self, value): + self._c2s_no_context_takeover = value + + def enable_outgoing_compression(self): + self._compress_outgoing_enabled = True + + def disable_outgoing_compression(self): + self._compress_outgoing_enabled = False + + def _process_incoming_message(self, message, decompress): + if not decompress: + return message + + received_payload_size = len(message) + self._total_incoming_payload_bytes += received_payload_size + + message = self._inflater.filter(message) + + filtered_payload_size = len(message) + self._total_filtered_incoming_payload_bytes += filtered_payload_size + + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) + + return message + + def _process_outgoing_message(self, message, end, binary): + if not binary: + message = message.encode('utf-8') + + if not self._compress_outgoing_enabled: + return message + + original_payload_size = len(message) + self._total_outgoing_payload_bytes += original_payload_size + + message = self._deflater.filter(message, flush=end) + + filtered_payload_size = len(message) + self._total_filtered_outgoing_payload_bytes += filtered_payload_size + + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) + + if not self._compress_ongoing: + self._outgoing_frame_filter.set_compression_bit() + self._compress_ongoing = not end + return message + + def _process_incoming_frame(self, frame): + if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): + self._incoming_message_filter.decompress_next_message() + frame.rsv1 = 0 + + def _process_outgoing_frame(self, frame, compression_bit): + if (not compression_bit or + common.is_control_opcode(frame.opcode)): + return + + frame.rsv1 = 1 + + +class PerMessageCompressionExtensionProcessor( + CompressionExtensionProcessorBase): + """WebSocket Per-message compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERMESSAGE_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateMessageProcessor(method_desc) + return None + + +_available_processors[common.PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +# Adding vendor-prefixed permessage-compress extension. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. +_available_processors[common.X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +class MuxExtensionProcessor(ExtensionProcessorInterface): + """WebSocket multiplexing extension processor.""" + + _QUOTA_PARAM = 'quota' + + def __init__(self, request): + self._request = request + + def name(self): + return common.MUX_EXTENSION + + def get_extension_response(self, ws_request, + logical_channel_extensions): + # Mux extension cannot be used after extensions that depend on + # frame boundary, extension data field, or any reserved bits + # which are attributed to each frame. + for extension in logical_channel_extensions: + name = extension.name() + if (name == common.PERFRAME_COMPRESSION_EXTENSION or + name == common.DEFLATE_FRAME_EXTENSION or + name == common.X_WEBKIT_DEFLATE_FRAME_EXTENSION): + return None + + quota = self._request.get_parameter_value(self._QUOTA_PARAM) + if quota is None: + ws_request.mux_quota = 0 + else: + try: + quota = int(quota) + except ValueError, e: + return None + if quota < 0 or quota >= 2 ** 32: + return None + ws_request.mux_quota = quota + + ws_request.mux = True + ws_request.mux_extensions = logical_channel_extensions + return common.ExtensionParameter(common.MUX_EXTENSION) + + def setup_stream_options(self, stream_options): + pass + + +_available_processors[common.MUX_EXTENSION] = MuxExtensionProcessor + + def get_extension_processor(extension_request): global _available_processors processor_class = _available_processors.get(extension_request.name()) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py index 10a178314..194f6b395 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py @@ -37,7 +37,6 @@ successfully established. import logging from mod_pywebsocket import common -from mod_pywebsocket.handshake import draft75 from mod_pywebsocket.handshake import hybi00 from mod_pywebsocket.handshake import hybi # Export AbortedByUserException, HandshakeException, and VersionException @@ -56,10 +55,8 @@ def do_handshake(request, dispatcher, allowDraft75=False, strict=False): Args: request: mod_python request. dispatcher: Dispatcher (dispatch.Dispatcher). - allowDraft75: allow draft 75 handshake protocol. - strict: Strictly check handshake request in draft 75. - Default: False. If True, request.connection must provide - get_memorized_lines method. + allowDraft75: obsolete argument. ignored. + strict: obsolete argument. ignored. Handshaker will add attributes such as ws_resource in performing handshake. @@ -86,9 +83,6 @@ def do_handshake(request, dispatcher, allowDraft75=False, strict=False): ('RFC 6455', hybi.Handshaker(request, dispatcher))) handshakers.append( ('HyBi 00', hybi00.Handshaker(request, dispatcher))) - if allowDraft75: - handshakers.append( - ('Hixie 75', draft75.Handshaker(request, dispatcher, strict))) for name, handshaker in handshakers: _LOGGER.debug('Trying protocol version %s', name) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py index bc095b129..e5c94ca90 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py @@ -85,13 +85,16 @@ def get_default_port(is_secure): def validate_subprotocol(subprotocol, hixie): - """Validate a value in subprotocol fields such as WebSocket-Protocol, - Sec-WebSocket-Protocol. + """Validate a value in the Sec-WebSocket-Protocol field. See - RFC 6455: Section 4.1., 4.2.2., and 4.3. - HyBi 00: Section 4.1. Opening handshake - - Hixie 75: Section 4.1. Handshake + + Args: + hixie: if True, checks if characters in subprotocol are in range + between U+0020 and U+007E. It's required by HyBi 00 but not by + RFC 6455. """ if not subprotocol: @@ -170,7 +173,11 @@ def check_request_line(request): # 5.1 1. The three character UTF-8 string "GET". # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). if request.method != 'GET': - raise HandshakeException('Method is not GET') + raise HandshakeException('Method is not GET: %r' % request.method) + + if request.protocol != 'HTTP/1.1': + raise HandshakeException('Version is not HTTP/1.1: %r' % + request.protocol) def check_header_lines(request, mandatory_headers): diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py deleted file mode 100644 index 802a31c9a..000000000 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket handshaking defined in draft-hixie-thewebsocketprotocol-75.""" - - -# Note: request.connection.write is used in this module, even though mod_python -# document says that it should be used only in connection handlers. -# Unfortunately, we have no other options. For example, request.write is not -# suitable because it doesn't allow direct raw bytes writing. - - -import logging -import re - -from mod_pywebsocket import common -from mod_pywebsocket.stream import StreamHixie75 -from mod_pywebsocket import util -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import build_location -from mod_pywebsocket.handshake._base import validate_subprotocol - - -_MANDATORY_HEADERS = [ - # key, expected value or None - ['Upgrade', 'WebSocket'], - ['Connection', 'Upgrade'], - ['Host', None], - ['Origin', None], -] - -_FIRST_FIVE_LINES = map(re.compile, [ - r'^GET /[\S]* HTTP/1.1\r\n$', - r'^Upgrade: WebSocket\r\n$', - r'^Connection: Upgrade\r\n$', - r'^Host: [\S]+\r\n$', - r'^Origin: [\S]+\r\n$', -]) - -_SIXTH_AND_LATER = re.compile( - r'^' - r'(WebSocket-Protocol: [\x20-\x7e]+\r\n)?' - r'(Cookie: [^\r]*\r\n)*' - r'(Cookie2: [^\r]*\r\n)?' - r'(Cookie: [^\r]*\r\n)*' - r'\r\n') - - -class Handshaker(object): - """This class performs WebSocket handshake.""" - - def __init__(self, request, dispatcher, strict=False): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - strict: Strictly check handshake request. Default: False. - If True, request.connection must provide get_memorized_lines - method. - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - self._strict = strict - - def do_handshake(self): - """Perform WebSocket Handshake. - - On _request, we set - ws_resource, ws_origin, ws_location, ws_protocol - ws_challenge_md5: WebSocket handshake information. - ws_stream: Frame generation/parsing class. - ws_version: Protocol version. - """ - - self._check_header_lines() - self._set_resource() - self._set_origin() - self._set_location() - self._set_subprotocol() - self._set_protocol_version() - - self._dispatcher.do_extra_handshake(self._request) - - self._send_handshake() - - self._logger.debug('Sent opening handshake response') - - def _set_resource(self): - self._request.ws_resource = self._request.uri - - def _set_origin(self): - self._request.ws_origin = self._request.headers_in['Origin'] - - def _set_location(self): - self._request.ws_location = build_location(self._request) - - def _set_subprotocol(self): - subprotocol = self._request.headers_in.get('WebSocket-Protocol') - if subprotocol is not None: - validate_subprotocol(subprotocol, hixie=True) - self._request.ws_protocol = subprotocol - - def _set_protocol_version(self): - self._logger.debug('IETF Hixie 75 protocol') - self._request.ws_version = common.VERSION_HIXIE75 - self._request.ws_stream = StreamHixie75(self._request) - - def _sendall(self, data): - self._request.connection.write(data) - - def _send_handshake(self): - self._sendall('HTTP/1.1 101 Web Socket Protocol Handshake\r\n') - self._sendall('Upgrade: WebSocket\r\n') - self._sendall('Connection: Upgrade\r\n') - self._sendall('WebSocket-Origin: %s\r\n' % self._request.ws_origin) - self._sendall('WebSocket-Location: %s\r\n' % self._request.ws_location) - if self._request.ws_protocol: - self._sendall( - 'WebSocket-Protocol: %s\r\n' % self._request.ws_protocol) - self._sendall('\r\n') - - def _check_header_lines(self): - for key, expected_value in _MANDATORY_HEADERS: - actual_value = self._request.headers_in.get(key) - if not actual_value: - raise HandshakeException('Header %s is not defined' % key) - if expected_value: - if actual_value != expected_value: - raise HandshakeException( - 'Expected %r for header %s but found %r' % - (expected_value, key, actual_value)) - if self._strict: - try: - lines = self._request.connection.get_memorized_lines() - except AttributeError, e: - raise AttributeError( - 'Strict handshake is specified but the connection ' - 'doesn\'t provide get_memorized_lines()') - self._check_first_lines(lines) - - def _check_first_lines(self, lines): - if len(lines) < len(_FIRST_FIVE_LINES): - raise HandshakeException('Too few header lines: %d' % len(lines)) - for line, regexp in zip(lines, _FIRST_FIVE_LINES): - if not regexp.search(line): - raise HandshakeException( - 'Unexpected header: %r doesn\'t match %r' - % (line, regexp.pattern)) - sixth_and_later = ''.join(lines[5:]) - if not _SIXTH_AND_LATER.search(sixth_and_later): - raise HandshakeException( - 'Unexpected header: %r doesn\'t match %r' - % (sixth_and_later, _SIXTH_AND_LATER.pattern)) - - -# vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py index 2883acbf8..fc0e2a096 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py @@ -182,34 +182,60 @@ class Handshaker(object): # Extra handshake handler may modify/remove processors. self._dispatcher.do_extra_handshake(self._request) + processors = filter(lambda processor: processor is not None, + self._request.ws_extension_processors) + + accepted_extensions = [] + + # We need to take care of mux extension here. Extensions that + # are placed before mux should be applied to logical channels. + mux_index = -1 + for i, processor in enumerate(processors): + if processor.name() == common.MUX_EXTENSION: + mux_index = i + break + if mux_index >= 0: + mux_processor = processors[mux_index] + logical_channel_processors = processors[:mux_index] + processors = processors[mux_index+1:] + + for processor in logical_channel_processors: + extension_response = processor.get_extension_response() + if extension_response is None: + # Rejected. + continue + accepted_extensions.append(extension_response) + # Pass a shallow copy of accepted_extensions as extensions for + # logical channels. + mux_response = mux_processor.get_extension_response( + self._request, accepted_extensions[:]) + if mux_response is not None: + accepted_extensions.append(mux_response) stream_options = StreamOptions() - self._request.ws_extensions = None - for processor in self._request.ws_extension_processors: - if processor is None: - # Some processors may be removed by extra handshake - # handler. - continue + # When there is mux extension, here, |processors| contain only + # prosessors for extensions placed after mux. + for processor in processors: extension_response = processor.get_extension_response() if extension_response is None: # Rejected. continue - if self._request.ws_extensions is None: - self._request.ws_extensions = [] - self._request.ws_extensions.append(extension_response) + accepted_extensions.append(extension_response) processor.setup_stream_options(stream_options) - if self._request.ws_extensions is not None: + if len(accepted_extensions) > 0: + self._request.ws_extensions = accepted_extensions self._logger.debug( 'Extensions accepted: %r', - map(common.ExtensionParameter.name, - self._request.ws_extensions)) + map(common.ExtensionParameter.name, accepted_extensions)) + else: + self._request.ws_extensions = None - self._request.ws_stream = Stream(self._request, stream_options) + self._request.ws_stream = self._create_stream(stream_options) if self._request.ws_requested_protocols is not None: if self._request.ws_protocol is None: @@ -268,7 +294,7 @@ class Handshaker(object): protocol_header = self._request.headers_in.get( common.SEC_WEBSOCKET_PROTOCOL_HEADER) - if not protocol_header: + if protocol_header is None: self._request.ws_requested_protocols = None return @@ -341,7 +367,10 @@ class Handshaker(object): return key - def _send_handshake(self, accept): + def _create_stream(self, stream_options): + return Stream(self._request, stream_options) + + def _create_handshake_response(self, accept): response = [] response.append('HTTP/1.1 101 Switching Protocols\r\n') @@ -363,7 +392,10 @@ class Handshaker(object): common.format_extensions(self._request.ws_extensions))) response.append('\r\n') - raw_response = ''.join(response) + return ''.join(response) + + def _send_handshake(self, accept): + raw_response = self._create_handshake_response(accept) self._request.connection.write(raw_response) self._logger.debug('Sent server\'s opening handshake: %r', raw_response) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py index b68c240e1..2cc62de04 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py @@ -63,8 +63,9 @@ _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT = ( _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION = { 'off': False, 'no': False, 'on': True, 'yes': True} -# PythonOption to specify to allow draft75 handshake. -# The default is None (Off) +# (Obsolete option. Ignored.) +# PythonOption to specify to allow handshake defined in Hixie 75 version +# protocol. The default is None (Off) _PYOPT_ALLOW_DRAFT75 = 'mod_pywebsocket.allow_draft75' # Map from values to their meanings. _PYOPT_ALLOW_DRAFT75_DEFINITION = {'off': False, 'on': True} diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py index 21ffdacf6..4c1a0114b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py @@ -59,20 +59,20 @@ def close_connection(request): request.ws_stream.close_connection() -def send_message(request, message, end=True, binary=False): - """Send message. +def send_message(request, payload_data, end=True, binary=False): + """Send a message (or part of a message). Args: request: mod_python request. - message: unicode text or str binary to send. - end: False to send message as a fragment. All messages until the - first call with end=True (inclusive) will be delivered to the - client in separate frames but as one WebSocket message. - binary: send message as binary frame. + payload_data: unicode text or str binary to send. + end: True to terminate a message. + False to send payload_data as part of a message that is to be + terminated by next or later send_message call with end=True. + binary: send payload_data as binary frame(s). Raises: BadOperationException: when server already terminated. """ - request.ws_stream.send_message(message, end, binary) + request.ws_stream.send_message(payload_data, end, binary) def receive_message(request): diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py new file mode 100644 index 000000000..f0bdd2461 --- /dev/null +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py @@ -0,0 +1,1636 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides classes and helper functions for multiplexing extension. + +Specification: +http://tools.ietf.org/html/draft-ietf-hybi-websocket-multiplexing-06 +""" + + +import collections +import copy +import email +import email.parser +import logging +import math +import struct +import threading +import traceback + +from mod_pywebsocket import common +from mod_pywebsocket import handshake +from mod_pywebsocket import util +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_hybi import Frame +from mod_pywebsocket._stream_hybi import Stream +from mod_pywebsocket._stream_hybi import StreamOptions +from mod_pywebsocket._stream_hybi import create_binary_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body +from mod_pywebsocket._stream_hybi import create_header +from mod_pywebsocket._stream_hybi import create_length_header +from mod_pywebsocket._stream_hybi import parse_frame +from mod_pywebsocket.handshake import hybi + + +_CONTROL_CHANNEL_ID = 0 +_DEFAULT_CHANNEL_ID = 1 + +_MUX_OPCODE_ADD_CHANNEL_REQUEST = 0 +_MUX_OPCODE_ADD_CHANNEL_RESPONSE = 1 +_MUX_OPCODE_FLOW_CONTROL = 2 +_MUX_OPCODE_DROP_CHANNEL = 3 +_MUX_OPCODE_NEW_CHANNEL_SLOT = 4 + +_MAX_CHANNEL_ID = 2 ** 29 - 1 + +_INITIAL_NUMBER_OF_CHANNEL_SLOTS = 64 +_INITIAL_QUOTA_FOR_CLIENT = 8 * 1024 + +_HANDSHAKE_ENCODING_IDENTITY = 0 +_HANDSHAKE_ENCODING_DELTA = 1 + +# We need only these status code for now. +_HTTP_BAD_RESPONSE_MESSAGES = { + common.HTTP_STATUS_BAD_REQUEST: 'Bad Request', +} + +# DropChannel reason code +# TODO(bashi): Define all reason code defined in -05 draft. +_DROP_CODE_NORMAL_CLOSURE = 1000 + +_DROP_CODE_INVALID_ENCAPSULATING_MESSAGE = 2001 +_DROP_CODE_CHANNEL_ID_TRUNCATED = 2002 +_DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED = 2003 +_DROP_CODE_UNKNOWN_MUX_OPCODE = 2004 +_DROP_CODE_INVALID_MUX_CONTROL_BLOCK = 2005 +_DROP_CODE_CHANNEL_ALREADY_EXISTS = 2006 +_DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION = 2007 + +_DROP_CODE_UNKNOWN_REQUEST_ENCODING = 3002 +_DROP_CODE_SEND_QUOTA_VIOLATION = 3005 +_DROP_CODE_ACKNOWLEDGED = 3008 + + +class MuxUnexpectedException(Exception): + """Exception in handling multiplexing extension.""" + pass + + +# Temporary +class MuxNotImplementedException(Exception): + """Raised when a flow enters unimplemented code path.""" + pass + + +class LogicalConnectionClosedException(Exception): + """Raised when logical connection is gracefully closed.""" + pass + + +class PhysicalConnectionError(Exception): + """Raised when there is a physical connection error.""" + def __init__(self, drop_code, message=''): + super(PhysicalConnectionError, self).__init__( + 'code=%d, message=%r' % (drop_code, message)) + self.drop_code = drop_code + self.message = message + + +class LogicalChannelError(Exception): + """Raised when there is a logical channel error.""" + def __init__(self, channel_id, drop_code, message=''): + super(LogicalChannelError, self).__init__( + 'channel_id=%d, code=%d, message=%r' % ( + channel_id, drop_code, message)) + self.channel_id = channel_id + self.drop_code = drop_code + self.message = message + + +def _encode_channel_id(channel_id): + if channel_id < 0: + raise ValueError('Channel id %d must not be negative' % channel_id) + + if channel_id < 2 ** 7: + return chr(channel_id) + if channel_id < 2 ** 14: + return struct.pack('!H', 0x8000 + channel_id) + if channel_id < 2 ** 21: + first = chr(0xc0 + (channel_id >> 16)) + return first + struct.pack('!H', channel_id & 0xffff) + if channel_id < 2 ** 29: + return struct.pack('!L', 0xe0000000 + channel_id) + + raise ValueError('Channel id %d is too large' % channel_id) + + +def _encode_number(number): + return create_length_header(number, False) + + +def _create_add_channel_response(channel_id, encoded_handshake, + encoding=0, rejected=False, + outer_frame_mask=False): + if encoding != 0 and encoding != 1: + raise ValueError('Invalid encoding %d' % encoding) + + first_byte = ((_MUX_OPCODE_ADD_CHANNEL_RESPONSE << 5) | + (rejected << 4) | encoding) + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(len(encoded_handshake)) + + encoded_handshake) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_drop_channel(channel_id, code=None, message='', + outer_frame_mask=False): + if len(message) > 0 and code is None: + raise ValueError('Code must be specified if message is specified') + + first_byte = _MUX_OPCODE_DROP_CHANNEL << 5 + block = chr(first_byte) + _encode_channel_id(channel_id) + if code is None: + block += _encode_number(0) # Reason size + else: + reason = struct.pack('!H', code) + message + reason_size = _encode_number(len(reason)) + block += reason_size + reason + + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_flow_control(channel_id, replenished_quota, + outer_frame_mask=False): + first_byte = _MUX_OPCODE_FLOW_CONTROL << 5 + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(replenished_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_new_channel_slot(slots, send_quota, outer_frame_mask=False): + if slots < 0 or send_quota < 0: + raise ValueError('slots and send_quota must be non-negative.') + first_byte = _MUX_OPCODE_NEW_CHANNEL_SLOT << 5 + block = (chr(first_byte) + + _encode_number(slots) + + _encode_number(send_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_fallback_new_channel_slot(outer_frame_mask=False): + first_byte = (_MUX_OPCODE_NEW_CHANNEL_SLOT << 5) | 1 # Set the F flag + block = (chr(first_byte) + _encode_number(0) + _encode_number(0)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _parse_request_text(request_text): + request_line, header_lines = request_text.split('\r\n', 1) + + words = request_line.split(' ') + if len(words) != 3: + raise ValueError('Bad Request-Line syntax %r' % request_line) + [command, path, version] = words + if version != 'HTTP/1.1': + raise ValueError('Bad request version %r' % version) + + # email.parser.Parser() parses RFC 2822 (RFC 822) style headers. + # RFC 6455 refers RFC 2616 for handshake parsing, and RFC 2616 refers + # RFC 822. + headers = email.parser.Parser().parsestr(header_lines) + return command, path, version, headers + + +class _ControlBlock(object): + """A structure that holds parsing result of multiplexing control block. + Control block specific attributes will be added by _MuxFramePayloadParser. + (e.g. encoded_handshake will be added for AddChannelRequest and + AddChannelResponse) + """ + + def __init__(self, opcode): + self.opcode = opcode + + +class _MuxFramePayloadParser(object): + """A class that parses multiplexed frame payload.""" + + def __init__(self, payload): + self._data = payload + self._read_position = 0 + self._logger = util.get_class_logger(self) + + def read_channel_id(self): + """Reads channel id. + + Raises: + ValueError: when the payload doesn't contain + valid channel id. + """ + + remaining_length = len(self._data) - self._read_position + pos = self._read_position + if remaining_length == 0: + raise ValueError('Invalid channel id format') + + channel_id = ord(self._data[pos]) + channel_id_length = 1 + if channel_id & 0xe0 == 0xe0: + if remaining_length < 4: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!L', + self._data[pos:pos+4])[0] & 0x1fffffff + channel_id_length = 4 + elif channel_id & 0xc0 == 0xc0: + if remaining_length < 3: + raise ValueError('Invalid channel id format') + channel_id = (((channel_id & 0x1f) << 16) + + struct.unpack('!H', self._data[pos+1:pos+3])[0]) + channel_id_length = 3 + elif channel_id & 0x80 == 0x80: + if remaining_length < 2: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!H', + self._data[pos:pos+2])[0] & 0x3fff + channel_id_length = 2 + self._read_position += channel_id_length + + return channel_id + + def read_inner_frame(self): + """Reads an inner frame. + + Raises: + PhysicalConnectionError: when the inner frame is invalid. + """ + + if len(self._data) == self._read_position: + raise PhysicalConnectionError( + _DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED) + + bits = ord(self._data[self._read_position]) + self._read_position += 1 + fin = (bits & 0x80) == 0x80 + rsv1 = (bits & 0x40) == 0x40 + rsv2 = (bits & 0x20) == 0x20 + rsv3 = (bits & 0x10) == 0x10 + opcode = bits & 0xf + payload = self.remaining_data() + # Consume rest of the message which is payload data of the original + # frame. + self._read_position = len(self._data) + return fin, rsv1, rsv2, rsv3, opcode, payload + + def _read_number(self): + if self._read_position + 1 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read the first byte of number field') + + number = ord(self._data[self._read_position]) + if number & 0x80 == 0x80: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'The most significant bit of the first byte of number should ' + 'be unset') + self._read_position += 1 + pos = self._read_position + if number == 127: + if pos + 8 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 8 + number = struct.unpack('!Q', self._data[pos:pos+8])[0] + if number > 0x7FFFFFFFFFFFFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Encoded number >= 2^63') + if number <= 0xFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 9 bytes encoding' % number) + return number + if number == 126: + if pos + 2 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 2 + number = struct.unpack('!H', self._data[pos:pos+2])[0] + if number <= 125: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 3 bytes encoding' % number) + return number + + def _read_size_and_contents(self): + """Reads data that consists of followings: + - the size of the contents encoded the same way as payload length + of the WebSocket Protocol with 1 bit padding at the head. + - the contents. + """ + + size = self._read_number() + pos = self._read_position + if pos + size > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read %d bytes data' % size) + + self._read_position += size + return self._data[pos:pos+size] + + def _read_add_channel_request(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x7 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + # Invalid encoding will be handled by MuxHandler. + encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoding = encoding + encoded_handshake = self._read_size_and_contents() + control_block.encoded_handshake = encoded_handshake + return control_block + + def _read_add_channel_response(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x3 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + control_block.accepted = (first_byte >> 4) & 1 + control_block.encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoded_handshake = self._read_size_and_contents() + return control_block + + def _read_flow_control(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.send_quota = self._read_number() + return control_block + + def _read_drop_channel(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + reason = self._read_size_and_contents() + if len(reason) == 0: + control_block.drop_code = None + control_block.drop_message = '' + elif len(reason) >= 2: + control_block.drop_code = struct.unpack('!H', reason[:2])[0] + control_block.drop_message = reason[2:] + else: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received DropChannel that conains only 1-byte reason') + return control_block + + def _read_new_channel_slot(self, first_byte, control_block): + reserved = first_byte & 0x1e + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + control_block.fallback = first_byte & 1 + control_block.slots = self._read_number() + control_block.send_quota = self._read_number() + return control_block + + def read_control_blocks(self): + """Reads control block(s). + + Raises: + PhysicalConnectionError: when the payload contains invalid control + block(s). + StopIteration: when no control blocks left. + """ + + while self._read_position < len(self._data): + first_byte = ord(self._data[self._read_position]) + self._read_position += 1 + opcode = (first_byte >> 5) & 0x7 + control_block = _ControlBlock(opcode=opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + yield self._read_add_channel_request(first_byte, control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + yield self._read_add_channel_response( + first_byte, control_block) + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + yield self._read_flow_control(first_byte, control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + yield self._read_drop_channel(first_byte, control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + yield self._read_new_channel_slot(first_byte, control_block) + else: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_MUX_OPCODE, + 'Invalid opcode %d' % opcode) + + assert self._read_position == len(self._data) + raise StopIteration + + def remaining_data(self): + """Returns remaining data.""" + + return self._data[self._read_position:] + + +class _LogicalRequest(object): + """Mimics mod_python request.""" + + def __init__(self, channel_id, command, path, protocol, headers, + connection): + """Constructs an instance. + + Args: + channel_id: the channel id of the logical channel. + command: HTTP request command. + path: HTTP request path. + headers: HTTP headers. + connection: _LogicalConnection instance. + """ + + self.channel_id = channel_id + self.method = command + self.uri = path + self.protocol = protocol + self.headers_in = headers + self.connection = connection + self.server_terminated = False + self.client_terminated = False + + def is_https(self): + """Mimics request.is_https(). Returns False because this method is + used only by old protocols (hixie and hybi00). + """ + + return False + + +class _LogicalConnection(object): + """Mimics mod_python mp_conn.""" + + # For details, see the comment of set_read_state(). + STATE_ACTIVE = 1 + STATE_GRACEFULLY_CLOSED = 2 + STATE_TERMINATED = 3 + + def __init__(self, mux_handler, channel_id): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + channel_id: channel id of this connection. + """ + + self._mux_handler = mux_handler + self._channel_id = channel_id + self._incoming_data = '' + self._write_condition = threading.Condition() + self._waiting_write_completion = False + self._read_condition = threading.Condition() + self._read_state = self.STATE_ACTIVE + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + + return self._mux_handler.physical_connection.get_local_addr() + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr.""" + + return self._mux_handler.physical_connection.get_remote_addr() + remote_addr = property(get_remote_addr) + + def get_memorized_lines(self): + """Gets memorized lines. Not supported.""" + + raise MuxUnexpectedException('_LogicalConnection does not support ' + 'get_memorized_lines') + + def write(self, data): + """Writes data. mux_handler sends data asynchronously. The caller will + be suspended until write done. + + Args: + data: data to be written. + + Raises: + MuxUnexpectedException: when called before finishing the previous + write. + """ + + try: + self._write_condition.acquire() + if self._waiting_write_completion: + raise MuxUnexpectedException( + 'Logical connection %d is already waiting the completion ' + 'of write' % self._channel_id) + + self._waiting_write_completion = True + self._mux_handler.send_data(self._channel_id, data) + self._write_condition.wait() + finally: + self._write_condition.release() + + def write_control_data(self, data): + """Writes data via the control channel. Don't wait finishing write + because this method can be called by mux dispatcher. + + Args: + data: data to be written. + """ + + self._mux_handler.send_control_data(data) + + def notify_write_done(self): + """Called when sending data is completed.""" + + try: + self._write_condition.acquire() + if not self._waiting_write_completion: + raise MuxUnexpectedException( + 'Invalid call of notify_write_done for logical connection' + ' %d' % self._channel_id) + self._waiting_write_completion = False + self._write_condition.notify() + finally: + self._write_condition.release() + + def append_frame_data(self, frame_data): + """Appends incoming frame data. Called when mux_handler dispatches + frame data to the corresponding application. + + Args: + frame_data: incoming frame data. + """ + + self._read_condition.acquire() + self._incoming_data += frame_data + self._read_condition.notify() + self._read_condition.release() + + def read(self, length): + """Reads data. Blocks until enough data has arrived via physical + connection. + + Args: + length: length of data to be read. + Raises: + LogicalConnectionClosedException: when closing handshake for this + logical channel has been received. + ConnectionTerminatedException: when the physical connection has + closed, or an error is caused on the reader thread. + """ + + self._read_condition.acquire() + while (self._read_state == self.STATE_ACTIVE and + len(self._incoming_data) < length): + self._read_condition.wait() + + try: + if self._read_state == self.STATE_GRACEFULLY_CLOSED: + raise LogicalConnectionClosedException( + 'Logical channel %d has closed.' % self._channel_id) + elif self._read_state == self.STATE_TERMINATED: + raise ConnectionTerminatedException( + 'Receiving %d byte failed. Logical channel (%d) closed' % + (length, self._channel_id)) + + value = self._incoming_data[:length] + self._incoming_data = self._incoming_data[length:] + finally: + self._read_condition.release() + + return value + + def set_read_state(self, new_state): + """Sets the state of this connection. Called when an event for this + connection has occurred. + + Args: + new_state: state to be set. new_state must be one of followings: + - STATE_GRACEFULLY_CLOSED: when closing handshake for this + connection has been received. + - STATE_TERMINATED: when the physical connection has closed or + DropChannel of this connection has received. + """ + + self._read_condition.acquire() + self._read_state = new_state + self._read_condition.notify() + self._read_condition.release() + + +class _LogicalStream(Stream): + """Mimics the Stream class. This class interprets multiplexed WebSocket + frames. + """ + + def __init__(self, request, send_quota, receive_quota): + """Constructs an instance. + + Args: + request: _LogicalRequest instance. + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + # TODO(bashi): Support frame filters. + stream_options = StreamOptions() + # Physical stream is responsible for masking. + stream_options.unmask_receive = False + # Control frames can be fragmented on logical channel. + stream_options.allow_fragmented_control_frame = True + Stream.__init__(self, request, stream_options) + self._send_quota = send_quota + self._send_quota_condition = threading.Condition() + self._receive_quota = receive_quota + self._write_inner_frame_semaphore = threading.Semaphore() + + def _create_inner_frame(self, opcode, payload, end=True): + # TODO(bashi): Support extensions that use reserved bits. + first_byte = (end << 7) | opcode + return (_encode_channel_id(self._request.channel_id) + + chr(first_byte) + payload) + + def _write_inner_frame(self, opcode, payload, end=True): + payload_length = len(payload) + write_position = 0 + + try: + # An inner frame will be fragmented if there is no enough send + # quota. This semaphore ensures that fragmented inner frames are + # sent in order on the logical channel. + # Note that frames that come from other logical channels or + # multiplexing control blocks can be inserted between fragmented + # inner frames on the physical channel. + self._write_inner_frame_semaphore.acquire() + while write_position < payload_length: + try: + self._send_quota_condition.acquire() + while self._send_quota == 0: + self._logger.debug( + 'No quota. Waiting FlowControl message for %d.' % + self._request.channel_id) + self._send_quota_condition.wait() + + remaining = payload_length - write_position + write_length = min(self._send_quota, remaining) + inner_frame_end = ( + end and + (write_position + write_length == payload_length)) + + inner_frame = self._create_inner_frame( + opcode, + payload[write_position:write_position+write_length], + inner_frame_end) + frame_data = self._writer.build( + inner_frame, end=True, binary=True) + self._send_quota -= write_length + self._logger.debug('Consumed quota=%d, remaining=%d' % + (write_length, self._send_quota)) + finally: + self._send_quota_condition.release() + + # Writing data will block the worker so we need to release + # _send_quota_condition before writing. + self._logger.debug('Sending inner frame: %r' % frame_data) + self._request.connection.write(frame_data) + write_position += write_length + + opcode = common.OPCODE_CONTINUATION + + except ValueError, e: + raise BadOperationException(e) + finally: + self._write_inner_frame_semaphore.release() + + def replenish_send_quota(self, send_quota): + """Replenish send quota.""" + + self._send_quota_condition.acquire() + self._send_quota += send_quota + self._logger.debug('Replenished send quota for channel id %d: %d' % + (self._request.channel_id, self._send_quota)) + self._send_quota_condition.notify() + self._send_quota_condition.release() + + def consume_receive_quota(self, amount): + """Consumes receive quota. Returns False on failure.""" + + if self._receive_quota < amount: + self._logger.debug('Violate quota on channel id %d: %d < %d' % + (self._request.channel_id, + self._receive_quota, amount)) + return False + self._receive_quota -= amount + return True + + def send_message(self, message, end=True, binary=False): + """Override Stream.send_message.""" + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + if binary and isinstance(message, unicode): + raise BadOperationException( + 'Message for binary frame must be instance of str') + + if binary: + opcode = common.OPCODE_BINARY + else: + opcode = common.OPCODE_TEXT + message = message.encode('utf-8') + + self._write_inner_frame(opcode, message, end) + + def _receive_frame(self): + """Overrides Stream._receive_frame. + + In addition to call Stream._receive_frame, this method adds the amount + of payload to receiving quota and sends FlowControl to the client. + We need to do it here because Stream.receive_message() handles + control frames internally. + """ + + opcode, payload, fin, rsv1, rsv2, rsv3 = Stream._receive_frame(self) + amount = len(payload) + self._receive_quota += amount + frame_data = _create_flow_control(self._request.channel_id, + amount) + self._logger.debug('Sending flow control for %d, replenished=%d' % + (self._request.channel_id, amount)) + self._request.connection.write_control_data(frame_data) + return opcode, payload, fin, rsv1, rsv2, rsv3 + + def receive_message(self): + """Overrides Stream.receive_message.""" + + # Just call Stream.receive_message(), but catch + # LogicalConnectionClosedException, which is raised when the logical + # connection has closed gracefully. + try: + return Stream.receive_message(self) + except LogicalConnectionClosedException, e: + self._logger.debug('%s', e) + return None + + def _send_closing_handshake(self, code, reason): + """Overrides Stream._send_closing_handshake.""" + + body = create_closing_handshake_body(code, reason) + self._logger.debug('Sending closing handshake for %d: (%r, %r)' % + (self._request.channel_id, code, reason)) + self._write_inner_frame(common.OPCODE_CLOSE, body, end=True) + + self._request.server_terminated = True + + def send_ping(self, body=''): + """Overrides Stream.send_ping""" + + self._logger.debug('Sending ping on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PING, body, end=True) + + self._ping_queue.append(body) + + def _send_pong(self, body): + """Overrides Stream._send_pong""" + + self._logger.debug('Sending pong on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PONG, body, end=True) + + def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): + """Overrides Stream.close_connection.""" + + # TODO(bashi): Implement + self._logger.debug('Closing logical connection %d' % + self._request.channel_id) + self._request.server_terminated = True + + def _drain_received_data(self): + """Overrides Stream._drain_received_data. Nothing need to be done for + logical channel. + """ + + pass + + +class _OutgoingData(object): + """A structure that holds data to be sent via physical connection and + origin of the data. + """ + + def __init__(self, channel_id, data): + self.channel_id = channel_id + self.data = data + + +class _PhysicalConnectionWriter(threading.Thread): + """A thread that is responsible for writing data to physical connection. + + TODO(bashi): Make sure there is no thread-safety problem when the reader + thread reads data from the same socket at a time. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + self._stop_requested = False + self._deque = collections.deque() + self._deque_condition = threading.Condition() + + def put_outgoing_data(self, data): + """Puts outgoing data. + + Args: + data: _OutgoingData instance. + + Raises: + BadOperationException: when the thread has been requested to + terminate. + """ + + try: + self._deque_condition.acquire() + if self._stop_requested: + raise BadOperationException('Cannot write data anymore') + + self._deque.append(data) + self._deque_condition.notify() + finally: + self._deque_condition.release() + + def _write_data(self, outgoing_data): + try: + self._mux_handler.physical_connection.write(outgoing_data.data) + except Exception, e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (self._mux_handler.physical_connection.remote_addr,), e) + raise + + # TODO(bashi): It would be better to block the thread that sends + # control data as well. + if outgoing_data.channel_id != _CONTROL_CHANNEL_ID: + self._mux_handler.notify_write_done(outgoing_data.channel_id) + + def run(self): + self._deque_condition.acquire() + while not self._stop_requested: + if len(self._deque) == 0: + self._deque_condition.wait() + continue + + outgoing_data = self._deque.popleft() + self._deque_condition.release() + self._write_data(outgoing_data) + self._deque_condition.acquire() + + # Flush deque + try: + while len(self._deque) > 0: + outgoing_data = self._deque.popleft() + self._write_data(outgoing_data) + finally: + self._deque_condition.release() + + def stop(self): + """Stops the writer thread.""" + + self._deque_condition.acquire() + self._stop_requested = True + self._deque_condition.notify() + self._deque_condition.release() + + +class _PhysicalConnectionReader(threading.Thread): + """A thread that is responsible for reading data from physical connection. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + + def run(self): + while True: + try: + physical_stream = self._mux_handler.physical_stream + message = physical_stream.receive_message() + if message is None: + break + # Below happens only when a data message is received. + opcode = physical_stream.get_last_received_opcode() + if opcode != common.OPCODE_BINARY: + self._mux_handler.fail_physical_connection( + _DROP_CODE_INVALID_ENCAPSULATING_MESSAGE, + 'Received a text message on physical connection') + break + + except ConnectionTerminatedException, e: + self._logger.debug('%s', e) + break + + try: + self._mux_handler.dispatch_message(message) + except PhysicalConnectionError, e: + self._mux_handler.fail_physical_connection( + e.drop_code, e.message) + break + except LogicalChannelError, e: + self._mux_handler.fail_logical_channel( + e.channel_id, e.drop_code, e.message) + except Exception, e: + self._logger.debug(traceback.format_exc()) + break + + self._mux_handler.notify_reader_done() + + +class _Worker(threading.Thread): + """A thread that is responsible for running the corresponding application + handler. + """ + + def __init__(self, mux_handler, request): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + request: _LogicalRequest instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self._request = request + self.setDaemon(True) + + def run(self): + self._logger.debug('Logical channel worker started. (id=%d)' % + self._request.channel_id) + try: + # Non-critical exceptions will be handled by dispatcher. + self._mux_handler.dispatcher.transfer_data(self._request) + finally: + self._mux_handler.notify_worker_done(self._request.channel_id) + + +class _MuxHandshaker(hybi.Handshaker): + """Opening handshake processor for multiplexing.""" + + _DUMMY_WEBSOCKET_KEY = 'dGhlIHNhbXBsZSBub25jZQ==' + + def __init__(self, request, dispatcher, send_quota, receive_quota): + """Constructs an instance. + Args: + request: _LogicalRequest instance. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + hybi.Handshaker.__init__(self, request, dispatcher) + self._send_quota = send_quota + self._receive_quota = receive_quota + + # Append headers which should not be included in handshake field of + # AddChannelRequest. + # TODO(bashi): Make sure whether we should raise exception when + # these headers are included already. + request.headers_in[common.UPGRADE_HEADER] = ( + common.WEBSOCKET_UPGRADE_TYPE) + request.headers_in[common.CONNECTION_HEADER] = ( + common.UPGRADE_CONNECTION_TYPE) + request.headers_in[common.SEC_WEBSOCKET_VERSION_HEADER] = ( + str(common.VERSION_HYBI_LATEST)) + request.headers_in[common.SEC_WEBSOCKET_KEY_HEADER] = ( + self._DUMMY_WEBSOCKET_KEY) + + def _create_stream(self, stream_options): + """Override hybi.Handshaker._create_stream.""" + + self._logger.debug('Creating logical stream for %d' % + self._request.channel_id) + return _LogicalStream(self._request, self._send_quota, + self._receive_quota) + + def _create_handshake_response(self, accept): + """Override hybi._create_handshake_response.""" + + response = [] + + response.append('HTTP/1.1 101 Switching Protocols\r\n') + + # Upgrade, Connection and Sec-WebSocket-Accept should be excluded. + if self._request.ws_protocol is not None: + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + if (self._request.ws_extensions is not None and + len(self._request.ws_extensions) != 0): + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER, + common.format_extensions(self._request.ws_extensions))) + response.append('\r\n') + + return ''.join(response) + + def _send_handshake(self, accept): + """Override hybi.Handshaker._send_handshake.""" + + # Don't send handshake response for the default channel + if self._request.channel_id == _DEFAULT_CHANNEL_ID: + return + + handshake_response = self._create_handshake_response(accept) + frame_data = _create_add_channel_response( + self._request.channel_id, + handshake_response) + self._logger.debug('Sending handshake response for %d: %r' % + (self._request.channel_id, frame_data)) + self._request.connection.write_control_data(frame_data) + + +class _LogicalChannelData(object): + """A structure that holds information about logical channel. + """ + + def __init__(self, request, worker): + self.request = request + self.worker = worker + self.drop_code = _DROP_CODE_NORMAL_CLOSURE + self.drop_message = '' + + +class _HandshakeDeltaBase(object): + """A class that holds information for delta-encoded handshake.""" + + def __init__(self, headers): + self._headers = headers + + def create_headers(self, delta=None): + """Creates request headers for an AddChannelRequest that has + delta-encoded handshake. + + Args: + delta: headers should be overridden. + """ + + headers = copy.copy(self._headers) + if delta: + for key, value in delta.items(): + # The spec requires that a header with an empty value is + # removed from the delta base. + if len(value) == 0 and headers.has_key(key): + del headers[key] + else: + headers[key] = value + # TODO(bashi): Support extensions + headers['Sec-WebSocket-Extensions'] = '' + return headers + + +class _MuxHandler(object): + """Multiplexing handler. When a handler starts, it launches three + threads; the reader thread, the writer thread, and a worker thread. + + The reader thread reads data from the physical stream, i.e., the + ws_stream object of the underlying websocket connection. The reader + thread interprets multiplexed frames and dispatches them to logical + channels. Methods of this class are mostly called by the reader thread. + + The writer thread sends multiplexed frames which are created by + logical channels via the physical connection. + + The worker thread launched at the starting point handles the + "Implicitly Opened Connection". If multiplexing handler receives + an AddChannelRequest and accepts it, the handler will launch a new worker + thread and dispatch the request to it. + """ + + def __init__(self, request, dispatcher): + """Constructs an instance. + + Args: + request: mod_python request of the physical connection. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + """ + + self.original_request = request + self.dispatcher = dispatcher + self.physical_connection = request.connection + self.physical_stream = request.ws_stream + self._logger = util.get_class_logger(self) + self._logical_channels = {} + self._logical_channels_condition = threading.Condition() + # Holds client's initial quota + self._channel_slots = collections.deque() + self._handshake_base = None + self._worker_done_notify_received = False + self._reader = None + self._writer = None + + def start(self): + """Starts the handler. + + Raises: + MuxUnexpectedException: when the handler already started, or when + opening handshake of the default channel fails. + """ + + if self._reader or self._writer: + raise MuxUnexpectedException('MuxHandler already started') + + self._reader = _PhysicalConnectionReader(self) + self._writer = _PhysicalConnectionWriter(self) + self._reader.start() + self._writer.start() + + # Create "Implicitly Opened Connection". + logical_connection = _LogicalConnection(self, _DEFAULT_CHANNEL_ID) + self._handshake_base = _HandshakeDeltaBase( + self.original_request.headers_in) + logical_request = _LogicalRequest( + _DEFAULT_CHANNEL_ID, + self.original_request.method, + self.original_request.uri, + self.original_request.protocol, + self._handshake_base.create_headers(), + logical_connection) + # Client's send quota for the implicitly opened connection is zero, + # but we will send FlowControl later so set the initial quota to + # _INITIAL_QUOTA_FOR_CLIENT. + self._channel_slots.append(_INITIAL_QUOTA_FOR_CLIENT) + if not self._do_handshake_for_logical_request( + logical_request, send_quota=self.original_request.mux_quota): + raise MuxUnexpectedException( + 'Failed handshake on the default channel id') + self._add_logical_channel(logical_request) + + # Send FlowControl for the implicitly opened connection. + frame_data = _create_flow_control(_DEFAULT_CHANNEL_ID, + _INITIAL_QUOTA_FOR_CLIENT) + logical_request.connection.write_control_data(frame_data) + + def add_channel_slots(self, slots, send_quota): + """Adds channel slots. + + Args: + slots: number of slots to be added. + send_quota: initial send quota for slots. + """ + + self._channel_slots.extend([send_quota] * slots) + # Send NewChannelSlot to client. + frame_data = _create_new_channel_slot(slots, send_quota) + self.send_control_data(frame_data) + + def wait_until_done(self, timeout=None): + """Waits until all workers are done. Returns False when timeout has + occurred. Returns True on success. + + Args: + timeout: timeout in sec. + """ + + self._logical_channels_condition.acquire() + try: + while len(self._logical_channels) > 0: + self._logger.debug('Waiting workers(%d)...' % + len(self._logical_channels)) + self._worker_done_notify_received = False + self._logical_channels_condition.wait(timeout) + if not self._worker_done_notify_received: + self._logger.debug('Waiting worker(s) timed out') + return False + + finally: + self._logical_channels_condition.release() + + # Flush pending outgoing data + self._writer.stop() + self._writer.join() + + return True + + def notify_write_done(self, channel_id): + """Called by the writer thread when a write operation has done. + + Args: + channel_id: objective channel id. + """ + + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + channel_data.request.connection.notify_write_done() + else: + self._logger.debug('Seems that logical channel for %d has gone' + % channel_id) + finally: + self._logical_channels_condition.release() + + def send_control_data(self, data): + """Sends data via the control channel. + + Args: + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=_CONTROL_CHANNEL_ID, data=data)) + + def send_data(self, channel_id, data): + """Sends data via given logical channel. This method is called by + worker threads. + + Args: + channel_id: objective channel id. + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=channel_id, data=data)) + + def _send_drop_channel(self, channel_id, code=None, message=''): + frame_data = _create_drop_channel(channel_id, code, message) + self._logger.debug( + 'Sending drop channel for channel id %d' % channel_id) + self.send_control_data(frame_data) + + def _send_error_add_channel_response(self, channel_id, status=None): + if status is None: + status = common.HTTP_STATUS_BAD_REQUEST + + if status in _HTTP_BAD_RESPONSE_MESSAGES: + message = _HTTP_BAD_RESPONSE_MESSAGES[status] + else: + self._logger.debug('Response message for %d is not found' % status) + message = '???' + + response = 'HTTP/1.1 %d %s\r\n\r\n' % (status, message) + frame_data = _create_add_channel_response(channel_id, + encoded_handshake=response, + encoding=0, rejected=True) + self.send_control_data(frame_data) + + def _create_logical_request(self, block): + if block.channel_id == _CONTROL_CHANNEL_ID: + # TODO(bashi): Raise PhysicalConnectionError with code 2006 + # instead of MuxUnexpectedException. + raise MuxUnexpectedException( + 'Received the control channel id (0) as objective channel ' + 'id for AddChannel') + + if block.encoding > _HANDSHAKE_ENCODING_DELTA: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_REQUEST_ENCODING) + + method, path, version, headers = _parse_request_text( + block.encoded_handshake) + if block.encoding == _HANDSHAKE_ENCODING_DELTA: + headers = self._handshake_base.create_headers(headers) + + connection = _LogicalConnection(self, block.channel_id) + request = _LogicalRequest(block.channel_id, method, path, version, + headers, connection) + return request + + def _do_handshake_for_logical_request(self, request, send_quota=0): + try: + receive_quota = self._channel_slots.popleft() + except IndexError: + raise LogicalChannelError( + request.channel_id, _DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION) + + handshaker = _MuxHandshaker(request, self.dispatcher, + send_quota, receive_quota) + try: + handshaker.do_handshake() + except handshake.VersionException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response( + request.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return False + except handshake.HandshakeException, e: + # TODO(bashi): Should we _Fail the Logical Channel_ with 3001 + # instead? + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id, + status=e.status) + return False + except handshake.AbortedByUserException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id) + return False + + return True + + def _add_logical_channel(self, logical_request): + try: + self._logical_channels_condition.acquire() + if logical_request.channel_id in self._logical_channels: + self._logger.debug('Channel id %d already exists' % + logical_request.channel_id) + raise PhysicalConnectionError( + _DROP_CODE_CHANNEL_ALREADY_EXISTS, + 'Channel id %d already exists' % + logical_request.channel_id) + worker = _Worker(self, logical_request) + channel_data = _LogicalChannelData(logical_request, worker) + self._logical_channels[logical_request.channel_id] = channel_data + worker.start() + finally: + self._logical_channels_condition.release() + + def _process_add_channel_request(self, block): + try: + logical_request = self._create_logical_request(block) + except ValueError, e: + self._logger.debug('Failed to create logical request: %r' % e) + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return + if self._do_handshake_for_logical_request(logical_request): + if block.encoding == _HANDSHAKE_ENCODING_IDENTITY: + # Update handshake base. + # TODO(bashi): Make sure this is the right place to update + # handshake base. + self._handshake_base = _HandshakeDeltaBase( + logical_request.headers_in) + self._add_logical_channel(logical_request) + else: + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + + def _process_flow_control(self, block): + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.request.ws_stream.replenish_send_quota( + block.send_quota) + finally: + self._logical_channels_condition.release() + + def _process_drop_channel(self, block): + self._logger.debug( + 'DropChannel received for %d: code=%r, reason=%r' % + (block.channel_id, block.drop_code, block.drop_message)) + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.drop_code = _DROP_CODE_ACKNOWLEDGED + # Close the logical channel + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + finally: + self._logical_channels_condition.release() + + def _process_control_blocks(self, parser): + for control_block in parser.read_control_blocks(): + opcode = control_block.opcode + self._logger.debug('control block received, opcode: %d' % opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + self._process_add_channel_request(control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received AddChannelResponse') + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + self._process_flow_control(control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + self._process_drop_channel(control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received NewChannelSlot') + else: + raise MuxUnexpectedException( + 'Unexpected opcode %r' % opcode) + + def _process_logical_frame(self, channel_id, parser): + self._logger.debug('Received a frame. channel id=%d' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + # We must ignore the message for an inactive channel. + return + channel_data = self._logical_channels[channel_id] + fin, rsv1, rsv2, rsv3, opcode, payload = parser.read_inner_frame() + if not channel_data.request.ws_stream.consume_receive_quota( + len(payload)): + # The client violates quota. Close logical channel. + raise LogicalChannelError( + channel_id, _DROP_CODE_SEND_QUOTA_VIOLATION) + header = create_header(opcode, len(payload), fin, rsv1, rsv2, rsv3, + mask=False) + frame_data = header + payload + channel_data.request.connection.append_frame_data(frame_data) + finally: + self._logical_channels_condition.release() + + def dispatch_message(self, message): + """Dispatches message. The reader thread calls this method. + + Args: + message: a message that contains encapsulated frame. + Raises: + PhysicalConnectionError: if the message contains physical + connection level errors. + LogicalChannelError: if the message contains logical channel + level errors. + """ + + parser = _MuxFramePayloadParser(message) + try: + channel_id = parser.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_CHANNEL_ID_TRUNCATED) + if channel_id == _CONTROL_CHANNEL_ID: + self._process_control_blocks(parser) + else: + self._process_logical_frame(channel_id, parser) + + def notify_worker_done(self, channel_id): + """Called when a worker has finished. + + Args: + channel_id: channel id corresponded with the worker. + """ + + self._logger.debug('Worker for channel id %d terminated' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + raise MuxUnexpectedException( + 'Channel id %d not found' % channel_id) + channel_data = self._logical_channels.pop(channel_id) + finally: + self._worker_done_notify_received = True + self._logical_channels_condition.notify() + self._logical_channels_condition.release() + + if not channel_data.request.server_terminated: + self._send_drop_channel( + channel_id, code=channel_data.drop_code, + message=channel_data.drop_message) + + def notify_reader_done(self): + """This method is called by the reader thread when the reader has + finished. + """ + + # Terminate all logical connections + self._logger.debug('termiating all logical connections...') + self._logical_channels_condition.acquire() + for channel_data in self._logical_channels.values(): + try: + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + except Exception: + pass + self._logical_channels_condition.release() + + def fail_physical_connection(self, code, message): + """Fail the physical connection. + + Args: + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing the physical connection...') + self._send_drop_channel(_CONTROL_CHANNEL_ID, code, message) + self.physical_stream.close_connection( + common.STATUS_INTERNAL_ENDPOINT_ERROR) + + def fail_logical_channel(self, channel_id, code, message): + """Fail a logical channel. + + Args: + channel_id: channel id. + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing logical channel %d...' % channel_id) + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + # Close the logical channel. notify_worker_done() will be + # called later and it will send DropChannel. + channel_data.drop_code = code + channel_data.drop_message = message + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + else: + self._send_drop_channel(channel_id, code, message) + finally: + self._logical_channels_condition.release() + + +def use_mux(request): + return hasattr(request, 'mux') and request.mux + + +def start(request, dispatcher): + mux_handler = _MuxHandler(request, dispatcher) + mux_handler.start() + + mux_handler.add_channel_slots(_INITIAL_NUMBER_OF_CHANNEL_SLOTS, + _INITIAL_QUOTA_FOR_CLIENT) + + mux_handler.wait_until_done() + + +# vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py index 850aa5cd4..07a33d9c9 100755 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py @@ -32,27 +32,44 @@ """Standalone WebSocket server. +Use this file to launch pywebsocket without Apache HTTP Server. + + BASIC USAGE -Use this server to run mod_pywebsocket without Apache HTTP Server. +Go to the src directory and run -Usage: - python standalone.py [-p <ws_port>] [-w <websock_handlers>] - [-s <scan_dir>] - [-d <document_root>] - [-m <websock_handlers_map_file>] - ... for other options, see _main below ... + $ python mod_pywebsocket/standalone.py [-p <ws_port>] + [-w <websock_handlers>] + [-d <document_root>] <ws_port> is the port number to use for ws:// connection. <document_root> is the path to the root directory of HTML files. <websock_handlers> is the path to the root directory of WebSocket handlers. -See __init__.py for details of <websock_handlers> and how to write WebSocket -handlers. If this path is relative, <document_root> is used as the base. +If not specified, <document_root> will be used. See __init__.py (or +run $ pydoc mod_pywebsocket) for how to write WebSocket handlers. + +For more detail and other options, run + + $ python mod_pywebsocket/standalone.py --help + +or see _build_option_parser method below. + +For trouble shooting, adding "--log_level debug" might help you. + -<scan_dir> is a path under the root directory. If specified, only the -handlers under scan_dir are scanned. This is useful in saving scan time. +TRY DEMO + +Go to the src directory and run + + $ python standalone.py -d example + +to launch pywebsocket with the sample handler and html on port 80. Open +http://localhost/console.html, click the connect button, type something into +the text box next to the send button and click the send button. If everything +is working, you'll see the message you typed echoed by the server. SUPPORTING TLS @@ -63,10 +80,10 @@ To support TLS, run standalone.py with -t, -k, and -c options. SUPPORTING CLIENT AUTHENTICATION To support client authentication with TLS, run standalone.py with -t, -k, -c, -and --ca-certificate options. +and --tls-client-auth, and --tls-client-ca options. E.g., $./standalone.py -d ../example -p 10443 -t -c ../test/cert/cert.pem -k -../test/cert/key.pem --ca-certificate=../test/cert/cacert.pem +../test/cert/key.pem --tls-client-auth --tls-client-ca=../test/cert/cacert.pem CONFIGURATION FILE @@ -110,6 +127,7 @@ import CGIHTTPServer import SimpleHTTPServer import SocketServer import ConfigParser +import base64 import httplib import logging import logging.handlers @@ -224,6 +242,12 @@ class _StandaloneRequest(object): return self._request_handler.command method = property(get_method) + def get_protocol(self): + """Getter to mimic request.protocol.""" + + return self._request_handler.request_version + protocol = property(get_protocol) + def is_https(self): """Mimic request.is_https().""" @@ -264,6 +288,32 @@ class _StandaloneSSLConnection(object): return socket._fileobject(self._connection, mode, bufsize) +def _alias_handlers(dispatcher, websock_handlers_map_file): + """Set aliases specified in websock_handler_map_file in dispatcher. + + Args: + dispatcher: dispatch.Dispatcher instance + websock_handler_map_file: alias map file + """ + + fp = open(websock_handlers_map_file) + try: + for line in fp: + if line[0] == '#' or line.isspace(): + continue + m = re.match('(\S+)\s+(\S+)', line) + if not m: + logging.warning('Wrong format in map file:' + line) + continue + try: + dispatcher.add_resource_path_alias( + m.group(1), m.group(2)) + except dispatch.DispatchException, e: + logging.error(str(e)) + finally: + fp.close() + + class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): """HTTPServer specialized for WebSocket.""" @@ -278,6 +328,20 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): if necessary. """ + # Share a Dispatcher among request handlers to save time for + # instantiation. Dispatcher can be shared because it is thread-safe. + options.dispatcher = dispatch.Dispatcher( + options.websock_handlers, + options.scan_dir, + options.allow_handlers_outside_root_dir) + if options.websock_handlers_map_file: + _alias_handlers(options.dispatcher, + options.websock_handlers_map_file) + warnings = options.dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('mod_pywebsocket: %s' % warning) + self._logger = util.get_class_logger(self) self.request_queue_size = options.request_queue_size @@ -325,7 +389,7 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): continue if self.websocket_server_options.use_tls: if _HAS_SSL: - if self.websocket_server_options.ca_certificate: + if self.websocket_server_options.tls_client_auth: client_cert_ = ssl.CERT_REQUIRED else: client_cert_ = ssl.CERT_NONE @@ -333,7 +397,7 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): keyfile=self.websocket_server_options.private_key, certfile=self.websocket_server_options.certificate, ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.websocket_server_options.ca_certificate, + ca_certs=self.websocket_server_options.tls_client_ca, cert_reqs=client_cert_) if _HAS_OPEN_SSL: ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) @@ -362,6 +426,15 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): self._logger.info('Skip by failure: %r', e) socket_.close() failed_sockets.append(socketinfo) + if self.server_address[1] == 0: + # The operating system assigns the actual port number for port + # number 0. This case, the second and later sockets should use + # the same port number. Also self.server_port is rewritten + # because it is exported, and will be used by external code. + self.server_address = ( + self.server_name, socket_.getsockname()[1]) + self.server_port = self.server_address[1] + self._logger.info('Port %r is assigned', self.server_port) for socketinfo in failed_sockets: self._sockets.remove(socketinfo) @@ -386,6 +459,10 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): for socketinfo in failed_sockets: self._sockets.remove(socketinfo) + if len(self._sockets) == 0: + self._logger.critical( + 'No sockets activated. Use info log level to see the reason.') + def server_close(self): """Override SocketServer.TCPServer.server_close to enable multiple sockets close. @@ -513,6 +590,17 @@ class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): # attributes). if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): return False + + if self._options.use_basic_auth: + auth = self.headers.getheader('Authorization') + if auth != self._options.basic_auth_credential: + self.send_response(401) + self.send_header('WWW-Authenticate', + 'Basic realm="Pywebsocket"') + self.end_headers() + self._logger.info('Request basic authentication') + return True + host, port, resource = http_header_util.parse_uri(self.path) if resource is None: self._logger.info('Invalid URI: %r', self.path) @@ -648,32 +736,6 @@ def _configure_logging(options): deflate_log_level_name) -def _alias_handlers(dispatcher, websock_handlers_map_file): - """Set aliases specified in websock_handler_map_file in dispatcher. - - Args: - dispatcher: dispatch.Dispatcher instance - websock_handler_map_file: alias map file - """ - - fp = open(websock_handlers_map_file) - try: - for line in fp: - if line[0] == '#' or line.isspace(): - continue - m = re.match('(\S+)\s+(\S+)', line) - if not m: - logging.warning('Wrong format in map file:' + line) - continue - try: - dispatcher.add_resource_path_alias( - m.group(1), m.group(2)) - except dispatch.DispatchException, e: - logging.error(str(e)) - finally: - fp.close() - - def _build_option_parser(): parser = optparse.OptionParser() @@ -700,7 +762,9 @@ def _build_option_parser(): parser.add_option('-w', '--websock-handlers', '--websock_handlers', dest='websock_handlers', default='.', - help='WebSocket handlers root directory.') + help=('The root directory of WebSocket handler files. ' + 'If the path is relative, --document-root is used ' + 'as the base.')) parser.add_option('-m', '--websock-handlers-map-file', '--websock_handlers_map_file', dest='websock_handlers_map_file', @@ -710,15 +774,20 @@ def _build_option_parser(): 'existing_resource_path, separated by spaces.')) parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', default=None, - help=('WebSocket handlers scan directory. ' - 'Must be a directory under websock_handlers.')) + help=('Must be a directory under --websock-handlers. ' + 'Only handlers under this directory are scanned ' + 'and registered to the server. ' + 'Useful for saving scan time when the handler ' + 'root directory contains lots of files that are ' + 'not handler file or are handler files but you ' + 'don\'t want them to be registered. ')) parser.add_option('--allow-handlers-outside-root-dir', '--allow_handlers_outside_root_dir', dest='allow_handlers_outside_root_dir', action='store_true', default=False, help=('Scans WebSocket handlers even if their canonical ' - 'path is not under websock_handlers.')) + 'path is not under --websock-handlers.')) parser.add_option('-d', '--document-root', '--document_root', dest='document_root', default='.', help='Document root directory.') @@ -735,9 +804,20 @@ def _build_option_parser(): default='', help='TLS private key file.') parser.add_option('-c', '--certificate', dest='certificate', default='', help='TLS certificate file.') - parser.add_option('--ca-certificate', dest='ca_certificate', default='', - help=('TLS CA certificate file for client ' - 'authentication.')) + parser.add_option('--tls-client-auth', dest='tls_client_auth', + action='store_true', default=False, + help='Requires TLS client auth on every connection.') + parser.add_option('--tls-client-ca', dest='tls_client_ca', default='', + help=('Specifies a pem file which contains a set of ' + 'concatenated CA certificates which are used to ' + 'validate certificates passed from clients')) + parser.add_option('--basic-auth', dest='use_basic_auth', + action='store_true', default=False, + help='Requires Basic authentication.') + parser.add_option('--basic-auth-credential', + dest='basic_auth_credential', default='test:test', + help='Specifies the credential of basic authentication ' + 'by username:password pair (e.g. test:test).') parser.add_option('-l', '--log-file', '--log_file', dest='log_file', default='', help='Log file.') # Custom log level: @@ -771,9 +851,9 @@ def _build_option_parser(): help='Log backup count') parser.add_option('--allow-draft75', dest='allow_draft75', action='store_true', default=False, - help='Allow draft 75 handshake') + help='Obsolete option. Ignored.') parser.add_option('--strict', dest='strict', action='store_true', - default=False, help='Strictly check handshake request') + default=False, help='Obsolete option. Ignored.') parser.add_option('-q', '--queue', dest='request_queue_size', type='int', default=_DEFAULT_REQUEST_QUEUE_SIZE, help='request queue size') @@ -841,6 +921,12 @@ def _parse_args_and_config(args): def _main(args=None): + """You can call this function from your own program, but please note that + this function has some side-effects that might affect your program. For + example, util.wrap_popen3_for_win use in this method replaces implementation + of os.popen3. + """ + options, args = _parse_args_and_config(args=args) os.chdir(options.document_root) @@ -877,7 +963,7 @@ def _main(args=None): 'To use TLS, specify private_key and certificate.') sys.exit(1) - if options.ca_certificate: + if options.tls_client_auth: if not options.use_tls: logging.critical('TLS must be enabled for client authentication.') sys.exit(1) @@ -887,26 +973,16 @@ def _main(args=None): if not options.scan_dir: options.scan_dir = options.websock_handlers + if options.use_basic_auth: + options.basic_auth_credential = 'Basic ' + base64.b64encode( + options.basic_auth_credential) + try: if options.thread_monitor_interval_in_sec > 0: # Run a thread monitor to show the status of server threads for # debugging. ThreadMonitor(options.thread_monitor_interval_in_sec).start() - # Share a Dispatcher among request handlers to save time for - # instantiation. Dispatcher can be shared because it is thread-safe. - options.dispatcher = dispatch.Dispatcher( - options.websock_handlers, - options.scan_dir, - options.allow_handlers_outside_root_dir) - if options.websock_handlers_map_file: - _alias_handlers(options.dispatcher, - options.websock_handlers_map_file) - warnings = options.dispatcher.source_warnings() - if warnings: - for warning in warnings: - logging.warning('mod_pywebsocket: %s' % warning) - server = WebSocketServer(options) server.serve_forever() except Exception, e: diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py index d051eee20..edc533279 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py @@ -51,6 +51,7 @@ from mod_pywebsocket._stream_hybi import create_ping_frame from mod_pywebsocket._stream_hybi import create_pong_frame from mod_pywebsocket._stream_hybi import create_binary_frame from mod_pywebsocket._stream_hybi import create_text_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body # vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py index 6146e052f..56591d233 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py @@ -232,6 +232,12 @@ class _Deflater(object): self._compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) + def compress(self, bytes): + compressed_bytes = self._compress.compress(bytes) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + def compress_and_flush(self, bytes): compressed_bytes = self._compress.compress(bytes) compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) @@ -318,14 +324,15 @@ class _RFC1979Deflater(object): self._window_bits = window_bits self._no_context_takeover = no_context_takeover - def filter(self, bytes): - if self._deflater is None or self._no_context_takeover: + def filter(self, bytes, flush=True): + if self._deflater is None or (self._no_context_takeover and flush): self._deflater = _Deflater(self._window_bits) - # Strip last 4 octets which is LEN and NLEN field of a non-compressed - # block added for Z_SYNC_FLUSH. - return self._deflater.compress_and_flush(bytes)[:-4] - + if flush: + # Strip last 4 octets which is LEN and NLEN field of a + # non-compressed block added for Z_SYNC_FLUSH. + return self._deflater.compress_and_flush(bytes)[:-4] + return self._deflater.compress(bytes) class _RFC1979Inflater(object): """A decompressor class for byte sequence compressed and flushed following diff --git a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py index 79b729aad..b71f3daaf 100644 --- a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -285,7 +285,7 @@ Reason component: MOCK component cc: MOCK cc blocked: 50004 -MOCK reopen_bug 50004 with comment 'Re-opened since this is blocked by 60001' +MOCK reopen_bug 50004 with comment 'Re-opened since this is blocked by bug 60001' MOCK add_patch_to_bug: bug_id=60001, description=ROLLOUT of r3001, mark_for_review=False, mark_for_commit_queue=True, mark_for_landing=False -- Begin comment -- Any committer can land this patch automatically by marking it commit-queue+. The commit-queue will build and test the patch before landing to ensure that the rollout will be successful. This process takes approximately 15 minutes. diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py index 22fb1704b..4b63f2f67 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -54,17 +54,15 @@ class TestRebaseline(unittest.TestCase): self.assertEqual(command._baseline_directory("GTK Linux 32-bit Release"), "/mock-checkout/LayoutTests/platform/gtk") self.assertEqual(command._baseline_directory("EFL Linux 64-bit Debug"), "/mock-checkout/LayoutTests/platform/efl") self.assertEqual(command._baseline_directory("Qt Linux Release"), "/mock-checkout/LayoutTests/platform/qt") - self.assertEqual(command._baseline_directory("Webkit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac") - self.assertEqual(command._baseline_directory("Webkit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") + self.assertEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac") + self.assertEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") def test_rebaseline_updates_expectations_file_noop(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - # FIXME: work around the chromium skia expectations file to avoid getting a bunch of confusing warnings. - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") for path in lion_port.expectations_files(): tool.filesystem.write_text_file(path, '') tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] @@ -74,11 +72,11 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "fast/css/large-list-of-rules-crash.html"), "Dummy test contents") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) self.assertEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] @@ -90,54 +88,52 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "Bug(x) [ SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") + self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") def test_rebaseline_does_not_include_overrides(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), "Bug(y) [ Mac ] other-test.html [ Failure ]\n") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "Bug(x) [ SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") + self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") def test_rebaseline_test(self): command = RebaselineTest() command.bind_to_tool(MockTool()) - expected_logs = "Retrieving http://example.com/f/builders/Webkit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) + expected_logs = "Retrieving http://example.com/f/builders/WebKit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) def test_rebaseline_test_and_print_scm_changes(self): command = RebaselineTest() command.bind_to_tool(MockTool()) - expected_logs = "Retrieving http://example.com/f/builders/Webkit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" + expected_logs = "Retrieving http://example.com/f/builders/WebKit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" command._print_scm_changes = True command._scm_changes = {'add': [], 'delete': []} command._tool._scm.exists = lambda x: False - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) self.assertEquals(command._scm_changes, {'add': ['/mock-checkout/LayoutTests/platform/chromium-linux/userscripts/another-test-expected.txt'], 'delete': []}) def test_rebaseline_and_copy_test(self): @@ -145,13 +141,13 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Copying baseline from /mock-checkout/LayoutTests/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_test_no_existing_result(self): command = RebaselineTest() @@ -159,38 +155,38 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu command.bind_to_tool(tool) expected_logs = """No existing baseline for userscripts/another-test.html. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_test_with_lion_result(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Copying baseline from /mock-checkout/LayoutTests/platform/chromium-mac/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_no_overwrite_test(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") - snowleopard_port = tool.port_factory.get_from_builder_name("Webkit Mac10.6") + snowleopard_port = tool.port_factory.get_from_builder_name("WebKit Mac10.6") tool.filesystem.write_text_file(os.path.join(snowleopard_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Existing baseline at /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt, not copying over it. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_all(self): old_exact_matches = builders._exact_matches @@ -228,9 +224,7 @@ MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'user-scri tool = MockTool() command.bind_to_tool(tool) - # FIXME: work around the chromium skia expectations file to avoid getting a bunch of confusing warnings. - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") for port_name in tool.port_factory.all_port_names(): port = tool.port_factory.get(port_name) for path in port.expectations_files(): @@ -245,46 +239,9 @@ MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'user-scri tool.executive.run_in_parallel = run_in_parallel - expected_logs = """Retrieving results for chromium-linux-x86 from Webkit Linux 32. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-linux-x86_64 from Webkit Linux. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-mac-lion from Webkit Mac10.7. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-mac-snowleopard from Webkit Mac10.6. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-win-win7 from Webkit Win7. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-win-xp from Webkit Win. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for efl from EFL Linux 64-bit Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for gtk from GTK Linux 64-bit Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for mac-lion from Apple Lion Release WK1 (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for mac-mountainlion from Apple MountainLion Release WK1 (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for qt-linux from Qt Linux Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for win-7sp0 from Apple Win 7 Release (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -""" + expected_logs = "Retrieving results for chromium-linux-x86 from WebKit Linux 32.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-linux-x86_64 from WebKit Linux.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-mac-lion from WebKit Mac10.7.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-mac-snowleopard from WebKit Mac10.6.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-win-win7 from WebKit Win7.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-win-xp from WebKit XP.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for efl from EFL Linux 64-bit Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for gtk from GTK Linux 64-bit Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for mac-lion from Apple Lion Release WK1 (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for mac-mountainlion from Apple MountainLion Release WK1 (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for qt-linux from Qt Linux Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for win-7sp0 from Apple Win 7 Release (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\n" - expected_stdout = """[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')] -""" + expected_stdout = "[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit XP', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit XP', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')]\n" expected_stderr = """MOCK run_command: ['qmake', '-v'], cwd=None MOCK run_command: ['qmake', '-v'], cwd=None @@ -321,7 +278,6 @@ MOCK run_command: ['qmake', '-v'], cwd=None command.bind_to_tool(tool) port = tool.port_factory.get('chromium-mac-lion') - tool.filesystem.write_text_file(port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') for port_name in tool.port_factory.all_port_names(): port = tool.port_factory.get(port_name) diff --git a/Tools/Scripts/webkitpy/tool/main.py b/Tools/Scripts/webkitpy/tool/main.py index d1fde74b8..68348a05a 100755 --- a/Tools/Scripts/webkitpy/tool/main.py +++ b/Tools/Scripts/webkitpy/tool/main.py @@ -91,7 +91,7 @@ class WebKitPatch(MultiCommandTool, Host): # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. def handle_global_options(self, options): - self._initialize_scm(options.patch_directories) + self.initialize_scm(options.patch_directories) if options.status_host: self.status_server.set_host(options.status_host) if options.bot_id: diff --git a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py index 22e853491..6c64bdd7e 100644 --- a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py +++ b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py @@ -90,8 +90,8 @@ class BuildCoverageExtrapolatorTest(unittest.TestCase): port = host.port_factory.get('chromium-win-win7', None) converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros()) extrapolator = BuildCoverageExtrapolator(converter) - self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release')])) - self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win7"), set([ + self.assertEquals(extrapolator.extrapolate_test_configurations("WebKit XP"), set([TestConfiguration(version='xp', architecture='x86', build_type='release')])) + self.assertEquals(extrapolator.extrapolate_test_configurations("WebKit Win7"), set([ TestConfiguration(version='win7', architecture='x86', build_type='debug'), TestConfiguration(version='win7', architecture='x86', build_type='release')])) self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato") diff --git a/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py b/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py index 0e7458727..9e9c379d6 100644 --- a/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py +++ b/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py @@ -178,7 +178,7 @@ def get_test_baselines(test_file, test_config): # FIXME: This should get the Host from the test_config to be mockable! host = Host() - host._initialize_scm() + host.initialize_scm() host.filesystem = test_config.filesystem all_platforms_port = AllPlatformsPort(host) diff --git a/Tools/Scripts/webkitpy/tool/steps/createbug.py b/Tools/Scripts/webkitpy/tool/steps/createbug.py index 4605892e7..7e4a83504 100644 --- a/Tools/Scripts/webkitpy/tool/steps/createbug.py +++ b/Tools/Scripts/webkitpy/tool/steps/createbug.py @@ -53,4 +53,4 @@ class CreateBug(AbstractStep): if blocks: status = self._tool.bugs.fetch_bug(blocks).status() if status == 'RESOLVED': - self._tool.bugs.reopen_bug(blocks, "Re-opened since this is blocked by %s" % state["bug_id"]) + self._tool.bugs.reopen_bug(blocks, "Re-opened since this is blocked by bug %s" % state["bug_id"]) |