diff options
Diffstat (limited to 'deps/v8/tools/testrunner/local/testsuite.py')
| -rw-r--r-- | deps/v8/tools/testrunner/local/testsuite.py | 281 |
1 files changed, 172 insertions, 109 deletions
diff --git a/deps/v8/tools/testrunner/local/testsuite.py b/deps/v8/tools/testrunner/local/testsuite.py index 3b8f956a7f..946e89a3fc 100644 --- a/deps/v8/tools/testrunner/local/testsuite.py +++ b/deps/v8/tools/testrunner/local/testsuite.py @@ -50,15 +50,17 @@ class VariantGenerator(object): def FilterVariantsByTest(self, testcase): result = self.all_variants - if testcase.outcomes: - if statusfile.OnlyStandardVariant(testcase.outcomes): + outcomes = testcase.suite.GetStatusFileOutcomes(testcase) + if outcomes: + if statusfile.OnlyStandardVariant(outcomes): return self.standard_variant - if statusfile.OnlyFastVariants(testcase.outcomes): + if statusfile.OnlyFastVariants(outcomes): result = self.fast_variants return result def GetFlagSets(self, testcase, variant): - if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): + outcomes = testcase.suite.GetStatusFileOutcomes(testcase) + if outcomes and statusfile.OnlyFastVariants(outcomes): return FAST_VARIANT_FLAGS[variant] else: return ALL_VARIANT_FLAGS[variant] @@ -86,12 +88,11 @@ class TestSuite(object): self.name = name # string self.root = root # string containing path self.tests = None # list of TestCase objects - self.rules = None # dictionary mapping test path to list of outcomes - self.wildcards = None # dictionary mapping test paths to list of outcomes + self.rules = None # {variant: {test name: [rule]}} + self.prefix_rules = None # {variant: {test name prefix: [rule]}} self.total_duration = None # float, assigned on demand - def shell(self): - return "d8" + self._outcomes_cache = dict() def suffix(self): return ".js" @@ -131,109 +132,104 @@ class TestSuite(object): """ pass - def DownloadData(self): - pass - def ReadStatusFile(self, variables): with open(self.status_file()) as f: - self.rules, self.wildcards = ( + self.rules, self.prefix_rules = ( statusfile.ReadStatusFile(f.read(), variables)) def ReadTestCases(self, context): self.tests = self.ListTests(context) - @staticmethod - def _FilterSlow(slow, mode): - return (mode == "run" and not slow) or (mode == "skip" and slow) + def GetStatusfileFlags(self, test): + """Gets runtime flags from a status file. - @staticmethod - def _FilterPassFail(pass_fail, mode): - return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail) - - def FilterTestCasesByStatus(self, warn_unused_rules, - slow_tests="dontcare", - pass_fail_tests="dontcare", - variants=False): - - # Use only variants-dependent rules and wildcards when filtering - # respective test cases and generic rules when filtering generic test - # cases. - if not variants: - rules = self.rules[""] - wildcards = self.wildcards[""] - else: - # We set rules and wildcards to a variant-specific version for each test - # below. - rules = {} - wildcards = {} + Every outcome that starts with "--" is a flag. Status file has to be loaded + before using this function. + """ + flags = [] + for outcome in self.GetStatusFileOutcomes(test): + if outcome.startswith('--'): + flags.append(outcome) + return flags - filtered = [] + def FilterTestCasesByStatus(self, + slow_tests_mode=None, + pass_fail_tests_mode=None): + """Filters tests by outcomes from status file. + + Status file has to be loaded before using this function. + + Args: + slow_tests_mode: What to do with slow tests. + pass_fail_tests_mode: What to do with pass or fail tests. - # Remember used rules as tuples of (rule, variant), where variant is "" for - # variant-independent rules. + Mode options: + None (default) - don't skip + "skip" - skip if slow/pass_fail + "run" - skip if not slow/pass_fail + """ + def _skip_slow(is_slow, mode): + return ( + (mode == 'run' and not is_slow) or + (mode == 'skip' and is_slow)) + + def _skip_pass_fail(pass_fail, mode): + return ( + (mode == 'run' and not pass_fail) or + (mode == 'skip' and pass_fail)) + + def _compliant(test): + outcomes = self.GetStatusFileOutcomes(test) + if statusfile.DoSkip(outcomes): + return False + if _skip_slow(statusfile.IsSlow(outcomes), slow_tests_mode): + return False + if _skip_pass_fail(statusfile.IsPassOrFail(outcomes), + pass_fail_tests_mode): + return False + return True + + self.tests = filter(_compliant, self.tests) + + def WarnUnusedRules(self, check_variant_rules=False): + """Finds and prints unused rules in status file. + + Rule X is unused when it doesn't apply to any tests, which can also mean + that all matching tests were skipped by another rule before evaluating X. + + Status file has to be loaded before using this function. + """ + + if check_variant_rules: + variants = list(ALL_VARIANTS) + else: + variants = [''] used_rules = set() for t in self.tests: - slow = False - pass_fail = False testname = self.CommonTestName(t) variant = t.variant or "" - if variants: - rules = self.rules[variant] - wildcards = self.wildcards[variant] - if testname in rules: - used_rules.add((testname, variant)) - # Even for skipped tests, as the TestCase object stays around and - # PrintReport() uses it. - t.outcomes = t.outcomes | rules[testname] - if statusfile.DoSkip(t.outcomes): - continue # Don't add skipped tests to |filtered|. - for outcome in t.outcomes: - if outcome.startswith('Flags: '): - t.flags += outcome[7:].split() - slow = statusfile.IsSlow(t.outcomes) - pass_fail = statusfile.IsPassOrFail(t.outcomes) - skip = False - for rule in wildcards: - assert rule[-1] == '*' - if testname.startswith(rule[:-1]): - used_rules.add((rule, variant)) - t.outcomes = t.outcomes | wildcards[rule] - if statusfile.DoSkip(t.outcomes): - skip = True - break # "for rule in wildcards" - slow = slow or statusfile.IsSlow(t.outcomes) - pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes) - if (skip - or self._FilterSlow(slow, slow_tests) - or self._FilterPassFail(pass_fail, pass_fail_tests)): - continue # "for t in self.tests" - filtered.append(t) - self.tests = filtered - - if not warn_unused_rules: - return - - if not variants: - for rule in self.rules[""]: - if (rule, "") not in used_rules: - print("Unused rule: %s -> %s (variant independent)" % ( - rule, self.rules[""][rule])) - for rule in self.wildcards[""]: - if (rule, "") not in used_rules: - print("Unused rule: %s -> %s (variant independent)" % ( - rule, self.wildcards[""][rule])) - else: - for variant in ALL_VARIANTS: - for rule in self.rules[variant]: - if (rule, variant) not in used_rules: - print("Unused rule: %s -> %s (variant: %s)" % ( - rule, self.rules[variant][rule], variant)) - for rule in self.wildcards[variant]: - if (rule, variant) not in used_rules: - print("Unused rule: %s -> %s (variant: %s)" % ( - rule, self.wildcards[variant][rule], variant)) + if testname in self.rules.get(variant, {}): + used_rules.add((testname, variant)) + if statusfile.DoSkip(self.rules[variant][testname]): + continue + + for prefix in self.prefix_rules.get(variant, {}): + if testname.startswith(prefix): + used_rules.add((prefix, variant)) + if statusfile.DoSkip(self.prefix_rules[variant][prefix]): + break + + for variant in variants: + for rule, value in (list(self.rules.get(variant, {}).iteritems()) + + list(self.prefix_rules.get(variant, {}).iteritems())): + if (rule, variant) not in used_rules: + if variant == '': + variant_desc = 'variant independent' + else: + variant_desc = 'variant: %s' % variant + print('Unused rule: %s -> %s (%s)' % (rule, value, variant_desc)) def FilterTestCasesByArgs(self, args): """Filter test cases based on command-line arguments. @@ -260,7 +256,66 @@ class TestSuite(object): break self.tests = filtered - def GetFlagsForTestCase(self, testcase, context): + def GetExpectedOutcomes(self, testcase): + """Gets expected outcomes from status file. + + It differs from GetStatusFileOutcomes by selecting only outcomes that can + be result of test execution. + Status file has to be loaded before using this function. + """ + outcomes = self.GetStatusFileOutcomes(testcase) + + expected = [] + if (statusfile.FAIL in outcomes or + statusfile.FAIL_OK in outcomes): + expected.append(statusfile.FAIL) + + if statusfile.CRASH in outcomes: + expected.append(statusfile.CRASH) + + if statusfile.PASS in outcomes: + expected.append(statusfile.PASS) + + return expected or [statusfile.PASS] + + def GetStatusFileOutcomes(self, testcase): + """Gets outcomes from status file. + + Merges variant dependent and independent rules. Status file has to be loaded + before using this function. + """ + variant = testcase.variant or '' + testname = self.CommonTestName(testcase) + cache_key = '%s$%s' % (testname, variant) + + if cache_key not in self._outcomes_cache: + # Load statusfile to get outcomes for the first time. + assert(self.rules is not None) + assert(self.prefix_rules is not None) + + outcomes = frozenset() + + for key in set([variant, '']): + rules = self.rules.get(key, {}) + prefix_rules = self.prefix_rules.get(key, {}) + + if testname in rules: + outcomes |= rules[testname] + + for prefix in prefix_rules: + if testname.startswith(prefix): + outcomes |= prefix_rules[prefix] + + self._outcomes_cache[cache_key] = outcomes + + return self._outcomes_cache[cache_key] + + def GetShellForTestCase(self, testcase): + """Returns shell to be executed for this test case.""" + return 'd8' + + def GetParametersForTestCase(self, testcase, context): + """Returns a tuple of (files, flags, env) for this test case.""" raise NotImplementedError def GetSourceForTest(self, testcase): @@ -290,8 +345,7 @@ class TestSuite(object): return statusfile.PASS def HasUnexpectedOutput(self, testcase): - outcome = self.GetOutcome(testcase) - return not outcome in (testcase.outcomes or [statusfile.PASS]) + return self.GetOutcome(testcase) not in self.GetExpectedOutcomes(testcase) def StripOutputForTransmit(self, testcase): if not self.HasUnexpectedOutput(testcase): @@ -315,18 +369,24 @@ class GoogleTestSuite(TestSuite): super(GoogleTestSuite, self).__init__(name, root) def ListTests(self, context): - shell = os.path.abspath(os.path.join(context.shell_dir, self.shell())) + shell = os.path.abspath( + os.path.join(context.shell_dir, self.GetShellForTestCase(None))) if utils.IsWindows(): shell += ".exe" output = None for i in xrange(3): # Try 3 times in case of errors. - output = commands.Execute(context.command_prefix + - [shell, "--gtest_list_tests"] + - context.extra_flags) + cmd = ( + context.command_prefix + + [shell, "--gtest_list_tests"] + + context.extra_flags + ) + output = commands.Execute(cmd) if output.exit_code == 0: break - print "Test executable failed to list the tests (try %d).\n\nStdout:" % i + print "Test executable failed to list the tests (try %d).\n\nCmd:" % i + print ' '.join(cmd) + print "\nStdout:" print output.stdout print "\nStderr:" print output.stderr @@ -346,14 +406,17 @@ class GoogleTestSuite(TestSuite): tests.sort(key=lambda t: t.path) return tests - def GetFlagsForTestCase(self, testcase, context): - return (testcase.flags + ["--gtest_filter=" + testcase.path] + - ["--gtest_random_seed=%s" % context.random_seed] + - ["--gtest_print_time=0"] + - context.mode_flags) + def GetParametersForTestCase(self, testcase, context): + flags = ( + testcase.flags + + ["--gtest_filter=" + testcase.path] + + ["--gtest_random_seed=%s" % context.random_seed] + + ["--gtest_print_time=0"] + + context.mode_flags) + return [], flags, {} def _VariantGeneratorFactory(self): return StandardVariantGenerator - def shell(self): + def GetShellForTestCase(self, testcase): return self.name |