diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-12 14:27:29 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:35:20 +0000 |
commit | c30a6232df03e1efbd9f3b226777b07e087a1122 (patch) | |
tree | e992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/third_party/blink/tools/blinkpy/third_party | |
parent | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff) | |
download | qtwebengine-chromium-c30a6232df03e1efbd9f3b226777b07e087a1122.tar.gz |
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/third_party/blink/tools/blinkpy/third_party')
21 files changed, 453 insertions, 246 deletions
diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/README.chromium b/chromium/third_party/blink/tools/blinkpy/third_party/README.chromium index 84e3ccb7aeb..c797121def6 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/README.chromium +++ b/chromium/third_party/blink/tools/blinkpy/third_party/README.chromium @@ -22,7 +22,7 @@ Local Modifications: None Name: web-platform-tests - Test Suites for Web Platform specifications Short Name: wpt URL: https://github.com/web-platform-tests/wpt/ -Version: 622c9625dddfdef0c6dfafa8fa00d5119db50201 +Version: 66af89be218a1c81e89ad786104bf2866d006422 License: LICENSES FOR W3C TEST SUITES (https://www.w3.org/Consortium/Legal/2008/03-bsd-license.html) License File: wpt/wpt/LICENSE.md Security Critical: no @@ -32,4 +32,4 @@ Description: This includes code for the manifest tool, lint tool, and wptserve. web_tests/external/wpt contains the tests. Also see wpt/README.chromium for more details on maintenance. Local Modifications: -- Removed all files except for those listed in wpt/WPTWhiteList. +- Removed all files except for those listed in wpt/WPTIncludeList. diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/PRESUBMIT.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/PRESUBMIT.py index 9e559fe61d9..302b3fb3e33 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/PRESUBMIT.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/PRESUBMIT.py @@ -4,11 +4,14 @@ """Check the basic functionalities of WPT tools. -After rolling a new version of WPT tools, we should do some sanity checks. For -now, we only test `wpt lint` via web_tests/external/PRESUBMIT_test.py. +This PRESUBMIT guards against rolling a broken version of WPT tooling. It does +some smoke checks of WPT functionality. """ + def _TestWPTLint(input_api, output_api): + # We test 'wpt lint' by deferring to the web_tests/external presubmit test, + # which runs 'wpt lint' against web_tests/external/wpt. abspath_to_test = input_api.os_path.join( input_api.change.RepositoryRoot(), 'third_party', 'blink', 'web_tests', 'external', 'PRESUBMIT_test.py' @@ -24,9 +27,41 @@ def _TestWPTLint(input_api, output_api): return input_api.RunTests([command]) +def _TestWPTManifest(input_api, output_api): + # We test 'wpt manifest' by making a copy of the base manifest and updating + # it. A copy is used so that this PRESUBMIT doesn't change files in the tree. + blink_path = input_api.os_path.join( + input_api.change.RepositoryRoot(), 'third_party', 'blink') + + base_manifest = input_api.os_path.join( + blink_path, 'web_tests', 'external', 'WPT_BASE_MANIFEST_8.json') + with input_api.CreateTemporaryFile() as f: + f.write(input_api.ReadFile(base_manifest)) + f.close() + + wpt_exec_path = input_api.os_path.join( + blink_path, 'tools', 'blinkpy', 'third_party', 'wpt', 'wpt', 'wpt') + external_wpt = input_api.os_path.join( + blink_path, 'web_tests', 'external', 'wpt') + try: + input_api.subprocess.check_output( + ['python', wpt_exec_path, 'manifest', '--no-download', + '--path', f.name, '--tests-root', external_wpt]) + except input_api.subprocess.CalledProcessError as exc: + return [output_api.PresubmitError('wpt manifest failed:', long_text=exc.output)] + + return [] + + def CheckChangeOnUpload(input_api, output_api): - return _TestWPTLint(input_api, output_api) + results = [] + results += _TestWPTLint(input_api, output_api) + results += _TestWPTManifest(input_api, output_api) + return results def CheckChangeOnCommit(input_api, output_api): - return _TestWPTLint(input_api, output_api) + results = [] + results += _TestWPTLint(input_api, output_api) + results += _TestWPTManifest(input_api, output_api) + return results diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/README.chromium b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/README.chromium index be20e1fcf8a..84017a82f05 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/README.chromium +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/README.chromium @@ -35,7 +35,7 @@ checkout.sh Running this script without arguments will remove the existing checkout (third_party/wpt/wpt) and perform a fresh one. See "Rolling in WPT" for more. -WPTWhiteList +WPTIncludeList ============ The explicit list of files being kept, everything else not on this list is deleted when running "./checkout.sh reduce". Use this file to control what gets @@ -72,8 +72,8 @@ re-generate them: Rolling in WPT If there are new files that need to be rolled in, add the intended files to -the WPTWhiteList. Ensure these files are in the correct order by running -"LC_ALL=C sort WPTWhiteList". +the WPTIncludeList. Ensure these files are in the correct order by running +"LC_ALL=C sort WPTIncludeList". When rolling in new versions of WPT support, modify WPT_HEAD in checkout.sh to the desired HEAD position. You can then call "./checkout.sh clone" which will @@ -85,11 +85,11 @@ the "Local Modifications" section which lists ways in which Chromium has diverged from WPT. Make sure these modifications are persisted when reviewing the changes being made. -You can examine what's pulled in and update WPTWhiteList if some new files are +You can examine what's pulled in and update WPTIncludeList if some new files are required to run the updated version. Once you've cloned the repositories you can call "./checkout.sh reduce" to -remove everything that is not listed in WPTWhiteList. +remove everything that is not listed in WPTIncludeList. Note that calling "./checkout.sh" without arguments is equivalent of calling "./checkout.sh clone reduce". diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/WPTWhiteList b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/WPTIncludeList index b3d5da34dba..b3d5da34dba 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/WPTWhiteList +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/WPTIncludeList diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh index f2c0f0c8d1b..fb848e087d5 100755 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/checkout.sh @@ -1,15 +1,15 @@ #!/bin/bash # # Removes ./wpt/ directory containing the reduced web-platform-tests tree and -# starts a new checkout. Only files in WPTWhiteList are retained. The revisions -# getting checked out are defined in WPTHeads. +# starts a new checkout. Only files in WPTIncludeList are retained. The +# revisions getting checked out are defined in WPTHeads. DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) cd $DIR TARGET_DIR=$DIR/wpt REMOTE_REPO="https://github.com/web-platform-tests/wpt.git" -WPT_HEAD=622c9625dddfdef0c6dfafa8fa00d5119db50201 +WPT_HEAD=66af89be218a1c81e89ad786104bf2866d006422 function clone { # Remove existing repo if already exists. @@ -27,7 +27,7 @@ function reduce { # xargs on some platforms, so we remove those directories first. rm -fr html css # Remove all except white-listed. - comm -23 <(find . -type f | sort) <(cat ../WPTWhiteList | sort) | xargs -d '\n' -n 1 rm + comm -23 <(find . -type f | sort) <(cat ../WPTIncludeList | sort) | xargs -d '\n' -n 1 rm find . -empty -type d -delete } diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py index 74b3c8cf930..fc1a816f8ab 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/lint.py @@ -323,6 +323,41 @@ def check_css_globally_unique(repo_root, paths): return errors +def check_unique_testharness_basenames(repo_root, paths): + # type: (str, List[str]) -> List[rules.Error] + """ + Checks that all testharness files have unique basename paths. + + The 'basename path' refers to the entire path excluding the extension. For + example, 'foo/bar/baz.html' and 'foo/bar/baz.xhtml' have the same basename + path, but 'foo/bar/baz.html' and 'foo/qux/baz.html' do not. + + Testharness files with identical basenames have caused issues in downstream + infrastructure (see https://github.com/web-platform-tests/wpt/issues/7570), + and may cause confusion in general. + + :param repo_root: the repository root + :param paths: list of all paths + :returns: a list of errors found in ``paths`` + """ + + errors = [] + file_dict = defaultdict(list) + for path in paths: + source_file = SourceFile(repo_root, path, "/") + if source_file.type != "testharness": + continue + file_name, file_extension = os.path.splitext(path) + file_dict[file_name].append(file_extension) + for k, v in file_dict.items(): + if len(v) == 1: + continue + context = (', '.join(v),) + for extension in v: + errors.append(rules.DuplicateBasenamePath.error(k + extension, context)) + return errors + + def parse_ignorelist(f): # type: (IO[bytes]) -> Tuple[Ignorelist, Set[Text]] """ @@ -432,6 +467,7 @@ def check_parsed(repo_root, path, f): if (source_file.type != "support" and not source_file.name_is_reference and + not source_file.name_is_tentative and not source_file.spec_links): return [rules.MissingLink.error(path)] @@ -617,7 +653,7 @@ broken_python_metadata = re.compile(br"#\s*META:") def check_global_metadata(value): # type: (str) -> Iterable[Tuple[Type[rules.Rule], Tuple[Any, ...]]] - global_values = {item.strip() for item in value.split(b",") if item.strip()} + global_values = {item.strip().decode("utf8") for item in value.split(b",") if item.strip()} # TODO: this could check for duplicates and such for global_value in global_values: @@ -930,7 +966,7 @@ def lint(repo_root, paths, output_format, ignore_glob=str()): path_lints = [check_file_type, check_path_length, check_worker_collision, check_ahem_copy, check_gitignore_file] -all_paths_lints = [check_css_globally_unique] +all_paths_lints = [check_css_globally_unique, check_unique_testharness_basenames] file_lints = [check_regexp_line, check_parsed, check_python_ast, check_script_metadata, check_ahem_system_font] diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/rules.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/rules.py index 6fbdc1c360d..695f6cd4e53 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/rules.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/lint/rules.py @@ -322,6 +322,15 @@ class TestharnessInOtherType(Rule): description = "testharness.js included in a %s test" +class DuplicateBasenamePath(Rule): + name = "DUPLICATE-BASENAME-PATH" + description = collapse(""" + File has identical basename path (path excluding extension) as + other file(s) (found extensions: %s) + """) + to_fix = "rename files so they have unique basename paths" + + class Regexp(six.with_metaclass(abc.ABCMeta)): @abc.abstractproperty def pattern(self): diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/item.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/item.py index 72802758941..ee07f0d1d79 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/item.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/item.py @@ -191,6 +191,11 @@ class TestharnessTest(URLManifestItem): return self._extras.get("jsshell") @property + def quic(self): + # type: () -> Optional[bool] + return self._extras.get("quic") + + @property def script_metadata(self): # type: () -> Optional[Text] return self._extras.get("script_metadata") @@ -204,8 +209,10 @@ class TestharnessTest(URLManifestItem): rv[-1]["testdriver"] = self.testdriver if self.jsshell: rv[-1]["jsshell"] = True + if self.quic is not None: + rv[-1]["quic"] = self.quic if self.script_metadata: - rv[-1]["script_metadata"] = [(k.decode('utf8'), v.decode('utf8')) for (k,v) in self.script_metadata] + rv[-1]["script_metadata"] = [(k, v) for (k,v) in self.script_metadata] return rv diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/manifest.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/manifest.py index f79edc84352..345eeb6d2b0 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/manifest.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/manifest.py @@ -1,9 +1,18 @@ +import io import itertools import json import os from copy import deepcopy from multiprocessing import Pool, cpu_count -from six import PY3, iteritems, itervalues, string_types, binary_type, text_type +from six import ( + PY3, + binary_type, + ensure_text, + iteritems, + itervalues, + string_types, + text_type, +) from . import vcs from .item import (ConformanceCheckerTest, ManifestItem, ManualTest, RefTest, SupportFile, @@ -167,12 +176,14 @@ class Manifest(object): to_update = [] - for source_file, update in tree: + for source_file_or_path, update in tree: if not update: - assert isinstance(source_file, (binary_type, text_type)) - deleted.remove(tuple(source_file.split(os.path.sep))) + assert isinstance(source_file_or_path, (binary_type, text_type)) + path = ensure_text(source_file_or_path) + deleted.remove(tuple(path.split(os.path.sep))) else: - assert not isinstance(source_file, bytes) + assert not isinstance(source_file_or_path, (binary_type, text_type)) + source_file = source_file_or_path rel_path_parts = source_file.rel_path_parts assert isinstance(rel_path_parts, tuple) @@ -318,7 +329,7 @@ def _load(logger, # type: Logger else: logger.debug("Creating new manifest at %s" % manifest) try: - with open(manifest, "rb") as f: + with io.open(manifest, "r", encoding="utf-8") as f: rv = Manifest.from_json(tests_root, fast_json.load(f), types=types, diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py index d0794a2851e..299ad212f76 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/sourcefile.py @@ -2,7 +2,7 @@ import hashlib import re import os from collections import deque -from six import binary_type, iteritems, text_type +from six import binary_type, ensure_text, iteritems, text_type from six.moves.urllib.parse import urljoin from fnmatch import fnmatch @@ -56,9 +56,9 @@ def replace_end(s, old, new): def read_script_metadata(f, regexp): - # type: (BinaryIO, Pattern[bytes]) -> Iterable[Tuple[bytes, bytes]] + # type: (BinaryIO, Pattern[bytes]) -> Iterable[Tuple[Text, Text]] """ - Yields any metadata (pairs of bytestrings) from the file-like object `f`, + Yields any metadata (pairs of strings) from the file-like object `f`, as specified according to a supplied regexp. `regexp` - Regexp containing two groups containing the metadata name and @@ -70,25 +70,25 @@ def read_script_metadata(f, regexp): if not m: break - yield (m.groups()[0], m.groups()[1]) + yield (m.groups()[0].decode("utf8"), m.groups()[1].decode("utf8")) _any_variants = { - b"window": {"suffix": ".any.html"}, - b"serviceworker": {"force_https": True}, - b"sharedworker": {}, - b"dedicatedworker": {"suffix": ".any.worker.html"}, - b"worker": {"longhand": {b"dedicatedworker", b"sharedworker", b"serviceworker"}}, - b"jsshell": {"suffix": ".any.js"}, -} # type: Dict[bytes, Dict[str, Any]] + "window": {"suffix": ".any.html"}, + "serviceworker": {"force_https": True}, + "sharedworker": {}, + "dedicatedworker": {"suffix": ".any.worker.html"}, + "worker": {"longhand": {"dedicatedworker", "sharedworker", "serviceworker"}}, + "jsshell": {"suffix": ".any.js"}, +} # type: Dict[Text, Dict[Text, Any]] def get_any_variants(item): - # type: (bytes) -> Set[bytes] + # type: (Text) -> Set[Text] """ - Returns a set of variants (bytestrings) defined by the given keyword. + Returns a set of variants (strings) defined by the given keyword. """ - assert isinstance(item, binary_type), item + assert isinstance(item, text_type), item variant = _any_variants.get(item, None) if variant is None: @@ -98,46 +98,46 @@ def get_any_variants(item): def get_default_any_variants(): - # type: () -> Set[bytes] + # type: () -> Set[Text] """ - Returns a set of variants (bytestrings) that will be used by default. + Returns a set of variants (strings) that will be used by default. """ - return set({b"window", b"dedicatedworker"}) + return set({"window", "dedicatedworker"}) def parse_variants(value): - # type: (bytes) -> Set[bytes] + # type: (Text) -> Set[Text] """ - Returns a set of variants (bytestrings) defined by a comma-separated value. + Returns a set of variants (strings) defined by a comma-separated value. """ - assert isinstance(value, binary_type), value + assert isinstance(value, text_type), value - if value == b"": + if value == "": return get_default_any_variants() globals = set() - for item in value.split(b","): + for item in value.split(","): item = item.strip() globals |= get_any_variants(item) return globals def global_suffixes(value): - # type: (bytes) -> Set[Tuple[bytes, bool]] + # type: (Text) -> Set[Tuple[Text, bool]] """ Yields tuples of the relevant filename suffix (a string) and whether the variant is intended to run in a JS shell, for the variants defined by the given comma-separated value. """ - assert isinstance(value, binary_type), value + assert isinstance(value, text_type), value rv = set() global_types = parse_variants(value) for global_type in global_types: variant = _any_variants[global_type] - suffix = variant.get("suffix", ".any.%s.html" % global_type.decode("utf-8")) - rv.add((suffix, global_type == b"jsshell")) + suffix = variant.get("suffix", ".any.%s.html" % global_type) + rv.add((suffix, global_type == "jsshell")) return rv @@ -190,24 +190,21 @@ class SourceFile(object): ("css", "CSS2", "archive"), ("css", "common")} # type: Set[Tuple[bytes, ...]] - def __init__(self, tests_root, rel_path, url_base, hash=None, contents=None): + def __init__(self, tests_root, rel_path_str, url_base, hash=None, contents=None): # type: (AnyStr, AnyStr, Text, Optional[Text], Optional[bytes]) -> None """Object representing a file in a source tree. :param tests_root: Path to the root of the source tree - :param rel_path: File path relative to tests_root + :param rel_path_str: File path relative to tests_root :param url_base: Base URL used when converting file paths to urls :param contents: Byte array of the contents of the file or ``None``. """ + rel_path = ensure_text(rel_path_str) assert not os.path.isabs(rel_path), rel_path - if os.name == "nt": # do slash normalization on Windows - if isinstance(rel_path, binary_type): - rel_path = rel_path.replace(b"/", b"\\") - else: - rel_path = rel_path.replace(u"/", u"\\") + rel_path = rel_path.replace(u"/", u"\\") dir_path, filename = os.path.split(rel_path) name, ext = os.path.splitext(filename) @@ -218,13 +215,13 @@ class SourceFile(object): meta_flags = name.split(".")[1:] - self.tests_root = tests_root # type: Union[bytes, Text] - self.rel_path = rel_path # type: Union[bytes, Text] - self.dir_path = dir_path # type: Union[bytes, Text] - self.filename = filename # type: Union[bytes, Text] - self.name = name # type: Union[bytes, Text] - self.ext = ext # type: Union[bytes, Text] - self.type_flag = type_flag # type: Optional[Union[bytes, Text]] + self.tests_root = ensure_text(tests_root) # type: Text + self.rel_path = rel_path # type: Text + self.dir_path = dir_path # type: Text + self.filename = filename # type: Text + self.name = name # type: Text + self.ext = ext # type: Text + self.type_flag = type_flag # type: Optional[Text] self.meta_flags = meta_flags # type: Union[List[bytes], List[Text]] self.url_base = url_base self.contents = contents @@ -282,7 +279,7 @@ class SourceFile(object): @cached_property def path(self): - # type: () -> Union[bytes, Text] + # type: () -> Text return os.path.join(self.tests_root, self.rel_path) @cached_property @@ -411,6 +408,15 @@ class SourceFile(object): return self.type_flag == "crash" or "crashtests" in self.dir_path.split(os.path.sep) @property + def name_is_tentative(self): + # type: () -> bool + """Check if the file name matches the conditions for the file to be a + tentative file. + + See https://web-platform-tests.org/writing-tests/file-names.html#test-features""" + return "tentative" in self.meta_flags + + @property def markup_type(self): # type: () -> Optional[Text] """Return the type of markup contained in a file, based on its extension, @@ -462,7 +468,7 @@ class SourceFile(object): @cached_property def script_metadata(self): - # type: () -> Optional[List[Tuple[bytes, bytes]]] + # type: () -> Optional[List[Tuple[Text, Text]]] if self.name_is_worker or self.name_is_multi_global or self.name_is_window: regexp = js_meta_re elif self.name_is_webdriver: @@ -479,7 +485,7 @@ class SourceFile(object): """The timeout of a test or reference file. "long" if the file has an extended timeout or None otherwise""" if self.script_metadata: - if any(m == (b"timeout", b"long") for m in self.script_metadata): + if any(m == ("timeout", "long") for m in self.script_metadata): return "long" if self.root is None: @@ -641,8 +647,8 @@ class SourceFile(object): script_metadata = self.script_metadata assert script_metadata is not None for (key, value) in script_metadata: - if key == b"variant": - rv.append(value.decode("utf-8")) + if key == "variant": + rv.append(value) else: for element in self.variant_nodes: if "content" in element.attrib: @@ -675,6 +681,36 @@ class SourceFile(object): return bool(self.testdriver_nodes) @cached_property + def quic_nodes(self): + # type: () -> List[ElementTree.Element] + """List of ElementTree Elements corresponding to nodes in a test that + specify whether it needs QUIC server.""" + assert self.root is not None + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='quic']") + + @cached_property + def quic(self): + # type: () -> Optional[bool] + """Boolean indicating whether a test requires QUIC server + + Determined by <meta> elements (`quic_nodes()`) and "// META" comments + (`script_metadata()`). + """ + if self.script_metadata: + if any(m == ("quic", "true") for m in self.script_metadata): + return True + + if self.root is None: + return None + + if self.quic_nodes: + quic_str = self.quic_nodes[0].attrib.get("content", "false") # type: Text + if quic_str.lower() == "true": + return True + + return None + + @cached_property def reftest_nodes(self): # type: () -> List[ElementTree.Element] """List of ElementTree Elements corresponding to nodes representing a @@ -834,11 +870,11 @@ class SourceFile(object): )] elif self.name_is_multi_global: - globals = b"" + globals = u"" script_metadata = self.script_metadata assert script_metadata is not None for (key, value) in script_metadata: - if key == b"global": + if key == "global": globals = value break @@ -850,6 +886,7 @@ class SourceFile(object): global_variant_url(self.rel_url, suffix) + variant, timeout=self.timeout, jsshell=jsshell, + quic=self.quic, script_metadata=self.script_metadata ) for (suffix, jsshell) in sorted(global_suffixes(globals)) @@ -866,6 +903,7 @@ class SourceFile(object): self.url_base, test_url + variant, timeout=self.timeout, + quic=self.quic, script_metadata=self.script_metadata ) for variant in self.test_variants @@ -881,6 +919,7 @@ class SourceFile(object): self.url_base, test_url + variant, timeout=self.timeout, + quic=self.quic, script_metadata=self.script_metadata ) for variant in self.test_variants @@ -917,6 +956,7 @@ class SourceFile(object): self.url_base, url, timeout=self.timeout, + quic=self.quic, testdriver=testdriver, script_metadata=self.script_metadata )) @@ -930,6 +970,7 @@ class SourceFile(object): self.rel_url, references=self.references, timeout=self.timeout, + quic=self.quic, viewport_size=self.viewport_size, dpi=self.dpi, fuzzy=self.fuzzy @@ -957,9 +998,9 @@ class SourceFile(object): if drop_cached and "__cached_properties__" in self.__dict__: cached_properties = self.__dict__["__cached_properties__"] - for key in cached_properties: - if key in self.__dict__: - del self.__dict__[key] + for prop in cached_properties: + if prop in self.__dict__: + del self.__dict__[prop] del self.__dict__["__cached_properties__"] return rv diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/vcs.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/vcs.py index 05afdf39b22..7c0feeb8164 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/vcs.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/manifest/vcs.py @@ -226,12 +226,12 @@ class GitIgnoreCache(CacheFile, MutableMapping): # type: ignore def check_valid(self, data): # type: (Dict[Any, Any]) -> Dict[Any, Any] - ignore_path = os.path.join(self.tests_root, b".gitignore") + ignore_path = os.path.join(self.tests_root, ".gitignore") mtime = os.path.getmtime(ignore_path) - if data.get(b"/gitignore_file") != [ignore_path, mtime]: + if data.get("/gitignore_file") != [ignore_path, mtime]: self.modified = True data = {} - data[b"/gitignore_file"] = [ignore_path, mtime] + data["/gitignore_file"] = [ignore_path, mtime] return data def __contains__(self, key): diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/quic/requirements.txt b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/quic/requirements.txt index 165260c78f7..c66414cb751 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/quic/requirements.txt +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/quic/requirements.txt @@ -1 +1 @@ -aioquic==0.8.7 +aioquic==0.8.8 diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py index 45bfd4766eb..43ff7bd0dfb 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/serve/serve.py @@ -178,9 +178,9 @@ class HtmlWrapperHandler(WrapperHandler): def check_exposure(self, request): if self.global_type: - globals = b"" + globals = u"" for (key, value) in self._get_metadata(request): - if key == b"global": + if key == "global": globals = value break @@ -189,23 +189,23 @@ class HtmlWrapperHandler(WrapperHandler): self.global_type) def _meta_replacement(self, key, value): - if key == b"timeout": - if value == b"long": + if key == "timeout": + if value == "long": return '<meta name="timeout" content="long">' - if key == b"title": - value = value.decode('utf-8').replace("&", "&").replace("<", "<") + if key == "title": + value = value.replace("&", "&").replace("<", "<") return '<title>%s</title>' % value return None def _script_replacement(self, key, value): - if key == b"script": - attribute = value.decode('utf-8').replace("&", "&").replace('"', """) + if key == "script": + attribute = value.replace("&", "&").replace('"', """) return '<script src="%s"></script>' % attribute return None class WorkersHandler(HtmlWrapperHandler): - global_type = b"dedicatedworker" + global_type = "dedicatedworker" path_replace = [(".any.worker.html", ".any.js", ".any.worker.js"), (".worker.html", ".worker.js")] wrapper = """<!doctype html> @@ -234,7 +234,7 @@ class WindowHandler(HtmlWrapperHandler): class AnyHtmlHandler(HtmlWrapperHandler): - global_type = b"window" + global_type = "window" path_replace = [(".any.html", ".any.js")] wrapper = """<!doctype html> <meta charset=utf-8> @@ -254,7 +254,7 @@ self.GLOBAL = { class SharedWorkersHandler(HtmlWrapperHandler): - global_type = b"sharedworker" + global_type = "sharedworker" path_replace = [(".any.sharedworker.html", ".any.js", ".any.worker.js")] wrapper = """<!doctype html> <meta charset=utf-8> @@ -269,7 +269,7 @@ fetch_tests_from_worker(new SharedWorker("%(path)s%(query)s")); class ServiceWorkersHandler(HtmlWrapperHandler): - global_type = b"serviceworker" + global_type = "serviceworker" path_replace = [(".any.serviceworker.html", ".any.js", ".any.worker.js")] wrapper = """<!doctype html> <meta charset=utf-8> @@ -307,11 +307,11 @@ done(); return None def _script_replacement(self, key, value): - if key == b"script": - attribute = value.decode('utf-8').replace("\\", "\\\\").replace('"', '\\"') + if key == "script": + attribute = value.replace("\\", "\\\\").replace('"', '\\"') return 'importScripts("%s")' % attribute - if key == b"title": - value = value.decode('utf-8').replace("\\", "\\\\").replace('"', '\\"') + if key == "title": + value = value.replace("\\", "\\\\").replace('"', '\\"') return 'self.META_TITLE = "%s";' % value return None @@ -922,17 +922,22 @@ def run(**kwargs): signal.signal(signal.SIGTERM, handle_signal) signal.signal(signal.SIGINT, handle_signal) - while (all(item.is_alive() for item in iter_procs(servers)) and + while (all(subproc.is_alive() for subproc in iter_procs(servers)) and not received_signal.is_set()): - for item in iter_procs(servers): - item.join(1) - exited = [item for item in iter_procs(servers) if not item.is_alive()] - subject = "subprocess" if len(exited) == 1 else "subprocesses" - - logger.info("%s %s exited:" % (len(exited), subject)) - - for item in iter_procs(servers): - logger.info("Status of %s:\t%s" % (item.name, "running" if item.is_alive() else "not running")) + for subproc in iter_procs(servers): + subproc.join(1) + + failed_subproc = 0 + for subproc in iter_procs(servers): + if subproc.is_alive(): + logger.info('Status of subprocess "%s": running' % subproc.name) + else: + if subproc.exitcode == 0: + logger.info('Status of subprocess "%s": exited correctly' % subproc.name) + else: + logger.warning('Status of subprocess "%s": failed. Exit with non-zero status: %d' % (subproc.name, subproc.exitcode)) + failed_subproc += 1 + return failed_subproc def main(): diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py index 8b1b55f607f..366dc781d90 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/browser.py @@ -18,8 +18,7 @@ from .utils import call, get, untar, unzip uname = platform.uname() # the rootUrl for the firefox-ci deployment of Taskcluster -# (after November 9, https://firefox-ci-tc.services.mozilla.com/) -FIREFOX_CI_ROOT_URL = 'https://taskcluster.net' +FIREFOX_CI_ROOT_URL = 'https://firefox-ci-tc.services.mozilla.com' def _get_fileversion(binary, logger=None): @@ -49,6 +48,15 @@ def get_ext(filename): return ext +def get_taskcluster_artifact(index, path): + TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/" + + resp = get(TC_INDEX_BASE + "task/%s/artifacts/%s" % (index, path)) + resp.raise_for_status() + + return resp + + class Browser(object): __metaclass__ = ABCMeta @@ -174,7 +182,7 @@ class Firefox(Browser): url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel], os_builds[os_key]) self.logger.info("Downloading Firefox from %s" % url) - resp = requests.get(url) + resp = get(url) filename = None @@ -413,32 +421,32 @@ class Firefox(Browser): return find_executable(os.path.join(dest, "geckodriver")) def install_geckodriver_nightly(self, dest): - import tarfile - import mozdownload self.logger.info("Attempting to install webdriver from nightly") + + platform_bits = ("64" if uname[4] == "x86_64" else + ("32" if self.platform == "win" else "")) + tc_platform = "%s%s" % (self.platform, platform_bits) + + archive_ext = ".zip" if uname[0] == "Windows" else ".tar.gz" + archive_name = "public/geckodriver%s" % archive_ext + try: - s = mozdownload.DailyScraper(branch="mozilla-central", - extension="common.tests.tar.gz", - destination=dest) - package_path = s.download() - except mozdownload.errors.NotFoundError: + resp = get_taskcluster_artifact( + "gecko.v2.mozilla-central.latest.geckodriver.%s" % tc_platform, + archive_name) + except Exception: + self.logger.info("Geckodriver download failed") return - try: - exe_suffix = ".exe" if uname[0] == "Windows" else "" - with tarfile.open(package_path, "r") as f: - try: - member = f.getmember("bin%sgeckodriver%s" % (os.path.sep, - exe_suffix)) - except KeyError: - return - # Remove bin/ from the path. - member.name = os.path.basename(member.name) - f.extractall(members=[member], path=dest) - path = os.path.join(dest, member.name) - self.logger.info("Extracted geckodriver to %s" % path) - finally: - os.unlink(package_path) + if archive_ext == ".zip": + unzip(resp.raw, dest) + else: + untar(resp.raw, dest) + + exe_ext = ".exe" if uname[0] == "Windows" else "" + path = os.path.join(dest, "geckodriver%s" % exe_ext) + + self.logger.info("Extracted geckodriver to %s" % path) return path @@ -461,24 +469,10 @@ class FirefoxAndroid(Browser): if dest is None: dest = os.pwd - if FIREFOX_CI_ROOT_URL == 'https://taskcluster.net': - # NOTE: this condition can be removed after November 9, 2019 - TC_QUEUE_BASE = "https://queue.taskcluster.net/v1/" - TC_INDEX_BASE = "https://index.taskcluster.net/v1/" - else: - TC_QUEUE_BASE = FIREFOX_CI_ROOT_URL + "/api/queue/v1/" - TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/" - - - resp = requests.get(TC_INDEX_BASE + - "task/gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt") - resp.raise_for_status() - index = resp.json() - task_id = index["taskId"] - resp = requests.get(TC_QUEUE_BASE + "task/%s/artifacts/%s" % - (task_id, "public/build/geckoview-androidTest.apk")) - resp.raise_for_status() + resp = get_taskcluster_artifact( + "gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt", + "public/build/geckoview-androidTest.apk") filename = "geckoview-androidTest.apk" if rename: diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/commands.json b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/commands.json index 60fe1621af0..a47ab40d728 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/commands.json +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wpt/commands.json @@ -52,7 +52,6 @@ "parser": "get_parser", "help": "Install browser components", "install": [ - "mozdownload", "mozinstall" ] }, diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/pipes.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/pipes.py index 569875bf972..bbf25e6fe11 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/pipes.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/pipes.py @@ -320,6 +320,8 @@ class FirstWrapper(object): def __getitem__(self, key): try: + if isinstance(key, text_type): + key = key.encode('iso-8859-1') return self.params.first(key) except KeyError: return "" diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/request.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/request.py index a80bc0c8300..7c4d02d6de6 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/request.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/request.py @@ -1,13 +1,13 @@ import base64 import cgi -from six.moves.http_cookies import BaseCookie -from six import BytesIO, binary_type, text_type, iteritems, PY3 import tempfile +from six import BytesIO, binary_type, iteritems, PY3 +from six.moves.http_cookies import BaseCookie from six.moves.urllib.parse import parse_qsl, urlsplit from . import stash -from .utils import HTTPException +from .utils import HTTPException, isomorphic_encode, isomorphic_decode missing = object() @@ -219,22 +219,23 @@ class Request(object): MultiDict representing the parameters supplied with the request. Note that these may be present on non-GET requests; the name is chosen to be familiar to users of other systems such as PHP. + Both keys and values are binary strings. .. attribute:: POST MultiDict representing the request body parameters. Most parameters are present as string values, but file uploads have file-like - values. + values. All string values (including keys) have binary type. .. attribute:: cookies - Cookies object representing cookies sent with the request with a + A Cookies object representing cookies sent with the request with a dictionary-like interface. .. attribute:: auth - Object with username and password properties representing any - credentials supplied using HTTP authentication. + An instance of Authentication with username and password properties + representing any credentials supplied using HTTP authentication. .. attribute:: server @@ -248,8 +249,12 @@ class Request(object): self.protocol_version = request_handler.protocol_version self.method = request_handler.command + # Keys and values in raw headers are native strings. + self._headers = None + self.raw_headers = request_handler.headers + scheme = request_handler.server.scheme - host = request_handler.headers.get("Host") + host = self.raw_headers.get("Host") port = request_handler.server.server_address[1] if host is None: @@ -262,22 +267,17 @@ class Request(object): self.url_base = "/" if self.request_path.startswith(scheme + "://"): - self.url = request_handler.path + self.url = self.request_path else: - self.url = "%s://%s:%s%s" % (scheme, - host, - port, - self.request_path) + # TODO(#23362): Stop using native strings for URLs. + self.url = "%s://%s:%s%s" % ( + scheme, host, port, self.request_path) self.url_parts = urlsplit(self.url) - self.raw_headers = request_handler.headers - self.request_line = request_handler.raw_requestline - self._headers = None - self.raw_input = InputFile(request_handler.rfile, - int(self.headers.get("Content-Length", 0))) + int(self.raw_headers.get("Content-Length", 0))) self._body = None @@ -297,19 +297,24 @@ class Request(object): params = parse_qsl(self.url_parts.query, keep_blank_values=True) self._GET = MultiDict() for key, value in params: - self._GET.add(key, value) + self._GET.add(isomorphic_encode(key), isomorphic_encode(value)) return self._GET @property def POST(self): if self._POST is None: - #Work out the post parameters + # Work out the post parameters pos = self.raw_input.tell() self.raw_input.seek(0) - fs = cgi.FieldStorage(fp=self.raw_input, - environ={"REQUEST_METHOD": self.method}, - headers=self.raw_headers, - keep_blank_values=True) + kwargs = { + "fp": self.raw_input, + "environ": {"REQUEST_METHOD": self.method}, + "headers": self.raw_headers, + "keep_blank_values": True, + } + if PY3: + kwargs["encoding"] = "iso-8859-1" + fs = cgi.FieldStorage(**kwargs) self._POST = MultiDict.from_field_storage(fs) self.raw_input.seek(pos) return self._POST @@ -317,14 +322,12 @@ class Request(object): @property def cookies(self): if self._cookies is None: - parser = BaseCookie() + parser = BinaryCookieParser() cookie_headers = self.headers.get("cookie", b"") - if PY3: - cookie_headers = cookie_headers.decode("iso-8859-1") parser.load(cookie_headers) cookies = Cookies() for key, value in iteritems(parser): - cookies[key] = CookieValue(value) + cookies[isomorphic_encode(key)] = CookieValue(value) self._cookies = cookies return self._cookies @@ -357,24 +360,6 @@ class H2Request(Request): super(H2Request, self).__init__(request_handler) -def _maybe_encode(s): - """Encodes a text-type string into binary data using iso-8859-1. - - Returns `str` in Python 2 and `bytes` in Python 3. The function is a no-op - if the argument already has a binary type. - """ - if isinstance(s, binary_type): - return s - - # Python 3 assumes iso-8859-1 when parsing headers, which will garble text - # with non ASCII characters. We try to encode the text back to binary. - # https://github.com/python/cpython/blob/273fc220b25933e443c82af6888eb1871d032fb8/Lib/http/client.py#L213 - if isinstance(s, text_type): - return s.encode("iso-8859-1") - - raise TypeError("Unexpected value in RequestHeaders: %r" % s) - - class RequestHeaders(dict): """Read-only dictionary-like API for accessing request headers. @@ -384,7 +369,7 @@ class RequestHeaders(dict): """ def __init__(self, items): for header in items.keys(): - key = _maybe_encode(header).lower() + key = isomorphic_encode(header).lower() # get all headers with the same name values = items.getallmatchingheaders(header) if len(values) > 1: @@ -394,22 +379,21 @@ class RequestHeaders(dict): for value in values: # getallmatchingheaders returns raw header lines, so # split to get name, value - multiples.append(_maybe_encode(value).split(b':', 1)[1].strip()) + multiples.append(isomorphic_encode(value).split(b':', 1)[1].strip()) headers = multiples else: - headers = [_maybe_encode(items[header])] + headers = [isomorphic_encode(items[header])] dict.__setitem__(self, key, headers) - def __getitem__(self, key): """Get all headers of a certain (case-insensitive) name. If there is more than one, the values are returned comma separated""" - key = _maybe_encode(key) + key = isomorphic_encode(key) values = dict.__getitem__(self, key.lower()) if len(values) == 1: return values[0] else: - return ", ".join(values) + return b", ".join(values) def __setitem__(self, name, value): raise Exception @@ -430,7 +414,7 @@ class RequestHeaders(dict): def get_list(self, key, default=missing): """Get all the header values for a particular field name as a list""" - key = _maybe_encode(key) + key = isomorphic_encode(key) try: return dict.__getitem__(self, key.lower()) except KeyError: @@ -440,7 +424,7 @@ class RequestHeaders(dict): raise def __contains__(self, key): - key = _maybe_encode(key) + key = isomorphic_encode(key) return dict.__contains__(self, key.lower()) def iteritems(self): @@ -451,6 +435,7 @@ class RequestHeaders(dict): for item in self: yield self[item] + class CookieValue(object): """Representation of cookies. @@ -524,9 +509,8 @@ class CookieValue(object): class MultiDict(dict): - """Dictionary type that holds multiple values for each - key""" - #TODO: this should perhaps also order the keys + """Dictionary type that holds multiple values for each key""" + # TODO: this should perhaps also order the keys def __init__(self): pass @@ -541,7 +525,6 @@ class MultiDict(dict): def __getitem__(self, key): """Get the first value with a given key""" - #TODO: should this instead be the last value? return self.first(key) def first(self, key, default=missing): @@ -584,6 +567,10 @@ class MultiDict(dict): @classmethod def from_field_storage(cls, fs): + """Construct a MultiDict from a cgi.FieldStorage + + Note that all keys and values are binary strings. + """ self = cls() if fs.list is None: return self @@ -594,13 +581,46 @@ class MultiDict(dict): for value in values: if not value.filename: - value = value.value - self.add(key, value) + value = isomorphic_encode(value.value) + else: + assert isinstance(value, cgi.FieldStorage) + self.add(isomorphic_encode(key), value) return self +class BinaryCookieParser(BaseCookie): + """A subclass of BaseCookie that returns values in binary strings + + This is not intended to store the cookies; use Cookies instead. + """ + def value_decode(self, val): + """Decode value from network to (real_value, coded_value). + + Override BaseCookie.value_decode. + """ + return isomorphic_encode(val), val + + def value_encode(self, val): + raise NotImplementedError('BinaryCookieParser is not for setting cookies') + + def load(self, rawdata): + """Load cookies from a binary string. + + This overrides and calls BaseCookie.load. Unlike BaseCookie.load, it + does not accept dictionaries. + """ + assert isinstance(rawdata, binary_type) + if PY3: + # BaseCookie.load expects a native string, which in Python 3 is text. + rawdata = isomorphic_decode(rawdata) + super(BinaryCookieParser, self).load(rawdata) + + class Cookies(MultiDict): - """MultiDict specialised for Cookie values""" + """MultiDict specialised for Cookie values + + Keys and values are binary strings. + """ def __init__(self): pass diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/response.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/response.py index 3a60c2babb3..b6f27447451 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/response.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/response.py @@ -1,19 +1,22 @@ from collections import OrderedDict from datetime import datetime, timedelta -from six.moves.http_cookies import BaseCookie, Morsel +from io import BytesIO import json -import uuid import socket -from .constants import response_codes, h2_headers -from .logger import get_logger -from io import BytesIO +import uuid -from six import binary_type, text_type, integer_types, itervalues, PY3 -from hyperframe.frame import HeadersFrame, DataFrame, ContinuationFrame from hpack.struct import HeaderTuple +from hyperframe.frame import HeadersFrame, DataFrame, ContinuationFrame +from six import binary_type, text_type, integer_types, itervalues, PY3 +from six.moves.http_cookies import BaseCookie, Morsel + +from .constants import response_codes, h2_headers +from .logger import get_logger +from .utils import isomorphic_decode, isomorphic_encode missing = object() + class Response(object): """Object representing the response to a HTTP request @@ -79,7 +82,6 @@ class Response(object): self.headers = ResponseHeaders() self.content = [] - @property def status(self): return self._status @@ -99,8 +101,8 @@ class Response(object): """Set a cookie to be sent with a Set-Cookie header in the response - :param name: String name of the cookie - :param value: String value of the cookie + :param name: name of the cookie (a binary string) + :param value: value of the cookie (a binary string, or None) :param max_age: datetime.timedelta int representing the time (in seconds) until the cookie expires :param path: String path to which the cookie applies @@ -113,14 +115,20 @@ class Response(object): time or interval from now when the cookie expires """ + # TODO(Python 3): Convert other parameters (e.g. path) to bytes, too. + if value is None: + value = b'' + max_age = 0 + expires = timedelta(days=-1) + + if PY3: + name = isomorphic_decode(name) + value = isomorphic_decode(value) + days = {i+1: name for i, name in enumerate(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"])} - if value is None: - value = '' - max_age = 0 - expires = timedelta(days=-1) if isinstance(expires, timedelta): expires = datetime.utcnow() + expires @@ -154,11 +162,14 @@ class Response(object): def unset_cookie(self, name): """Remove a cookie from those that are being sent with the response""" + if PY3: + name = isomorphic_decode(name) cookies = self.headers.get("Set-Cookie") parser = BaseCookie() for cookie in cookies: if PY3: - cookie = cookie.decode("iso-8859-1") + # BaseCookie.load expects a text string. + cookie = isomorphic_decode(cookie) parser.load(cookie) if name in parser.keys(): @@ -223,9 +234,12 @@ class Response(object): self.write_status_headers() self.write_content() - def set_error(self, code, message=""): - """Set the response status headers and body to indicate an - error""" + def set_error(self, code, message=u""): + """Set the response status headers and return a JSON error object: + + {"error": {"code": code, "message": message}} + code is an int (HTTP status code), and message is a text string. + """ err = {"code": code, "message": message} data = json.dumps({"error": err}) @@ -297,24 +311,10 @@ class MultipartPart(object): def _maybe_encode(s): - """Encodes a text-type string into binary data using iso-8859-1. - - Returns `str` in Python 2 and `bytes` in Python 3. The function is a no-op - if the argument already has a binary type. - """ - if isinstance(s, binary_type): - return s - - # Python 3 assumes iso-8859-1 when parsing headers, which will garble text - # with non ASCII characters. We try to encode the text back to binary. - # https://github.com/python/cpython/blob/273fc220b25933e443c82af6888eb1871d032fb8/Lib/http/client.py#L213 - if isinstance(s, text_type): - return s.encode("iso-8859-1") - + """Encode a string or an int into binary data using isomorphic_encode().""" if isinstance(s, integer_types): return b"%i" % (s,) - - raise TypeError("Unexpected value in ResponseHeaders: %r" % s) + return isomorphic_encode(s) class ResponseHeaders(object): diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/sslutils/openssl.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/sslutils/openssl.py index aea1c7380b1..64f6d5fb2db 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/sslutils/openssl.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/sslutils/openssl.py @@ -402,6 +402,8 @@ class OpenSSLEnvironment(object): def _generate_host_cert(self, hosts): host = hosts[0] + if not self.force_regenerate: + self._load_ca_cert() if self._ca_key_path is None: self._generate_ca(hosts) ca_key_path = self._ca_key_path diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/stash.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/stash.py index 541aced6010..6b351847491 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/stash.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/stash.py @@ -4,7 +4,9 @@ import os import uuid import threading from multiprocessing.managers import AcquirerProxy, BaseManager, DictProxy -from six import text_type +from six import text_type, binary_type + +from .utils import isomorphic_encode class ServerDictManager(BaseManager): @@ -145,9 +147,12 @@ class Stash(object): if path is None: path = self.default_path # This key format is required to support using the path. Since the data - # passed into the stash can be a DictProxy which wouldn't detect changes - # when writing to a subdict. - return (str(path), str(uuid.UUID(key))) + # passed into the stash can be a DictProxy which wouldn't detect + # changes when writing to a subdict. + if isinstance(key, binary_type): + # UUIDs are within the ASCII charset. + key = key.decode('ascii') + return (isomorphic_encode(path), uuid.UUID(key).bytes) def put(self, key, value, path=None): """Place a value in the shared stash. diff --git a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/utils.py b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/utils.py index 64b08a27aa8..b005b417d74 100644 --- a/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/utils.py +++ b/chromium/third_party/blink/tools/blinkpy/third_party/wpt/wpt/tools/wptserve/wptserve/utils.py @@ -1,6 +1,45 @@ import socket import sys +from six import binary_type, text_type + + +def isomorphic_decode(s): + """Decodes a binary string into a text string using iso-8859-1. + + Returns `unicode` in Python 2 and `str` in Python 3. The function is a + no-op if the argument already has a text type. iso-8859-1 is chosen because + it is an 8-bit encoding whose code points range from 0x0 to 0xFF and the + values are the same as the binary representations, so any binary string can + be decoded into and encoded from iso-8859-1 without any errors or data + loss. Python 3 also uses iso-8859-1 (or latin-1) extensively in http: + https://github.com/python/cpython/blob/273fc220b25933e443c82af6888eb1871d032fb8/Lib/http/client.py#L213 + """ + if isinstance(s, text_type): + return s + + if isinstance(s, binary_type): + return s.decode("iso-8859-1") + + raise TypeError("Unexpected value (expecting string-like): %r" % s) + + +def isomorphic_encode(s): + """Encodes a text-type string into binary data using iso-8859-1. + + Returns `str` in Python 2 and `bytes` in Python 3. The function is a no-op + if the argument already has a binary type. This is the counterpart of + isomorphic_decode. + """ + if isinstance(s, binary_type): + return s + + if isinstance(s, text_type): + return s.encode("iso-8859-1") + + raise TypeError("Unexpected value (expecting string-like): %r" % s) + + def invert_dict(dict): rv = {} for key, values in dict.items(): @@ -25,6 +64,7 @@ def _open_socket(host, port): sock.listen(5) return sock + def is_bad_port(port): """ Bad port as per https://fetch.spec.whatwg.org/#port-blocking @@ -99,6 +139,7 @@ def is_bad_port(port): 6697, # irc+tls ] + def get_port(host=''): host = host or '127.0.0.1' port = 0 |