1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
#!/usr/bin/env vpython
import json
import optparse
import os
import sys
from blinkpy.common.host import Host
from blinkpy.web_tests.port import platform_options, configuration_options
def main(argv):
parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
parser.add_option('--failures', action='store_true',
help='show failing tests')
parser.add_option('--flakes', action='store_true',
help='show flaky tests')
parser.add_option('--expected', action='store_true',
help='include expected results along with unexpected')
parser.add_option('--passes', action='store_true',
help='show passing tests')
parser.add_option('--ignored-failures-path', action='store',
help='ignore failures seen in a previous run')
parser.add_options(platform_options())
parser.add_options(configuration_options())
options, args = parser.parse_args(argv)
host = Host()
if args:
if args[0] == '-':
txt = sys.stdin.read()
elif os.path.exists(args[0]):
with open(args[0], 'r') as fp:
txt = fp.read()
else:
print >> sys.stderr, "file not found: %s" % args[0]
sys.exit(1)
else:
txt = host.filesystem.read_text_file(
host.filesystem.join(
host.port_factory.get(
options=options).artifacts_directory(),
'full_results.json'))
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
passes, failures, flakes = decode_results(results, options.expected)
tests_to_print = []
if options.passes:
tests_to_print += passes.keys()
if options.failures:
tests_to_print += failures.keys()
if options.flakes:
tests_to_print += flakes.keys()
print "\n".join(sorted(tests_to_print))
if options.ignored_failures_path:
with open(options.ignored_failures_path, 'r') as fp:
txt = fp.read()
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
_, ignored_failures, _ = decode_results(results, options.expected)
new_failures = set(failures.keys()) - set(ignored_failures.keys())
if new_failures:
print "New failures:"
print "\n".join(sorted(new_failures))
print
if ignored_failures:
print "Ignored failures:"
print "\n".join(sorted(ignored_failures.keys()))
if new_failures:
return 1
return 0
def decode_results(results, include_expected=False):
tests = convert_trie_to_flat_paths(results['tests'])
failures = {}
flakes = {}
passes = {}
for (test, result) in tests.iteritems():
if include_expected or result.get('is_unexpected'):
actual_results = result['actual'].split()
expected_results = result['expected'].split()
if len(actual_results) > 1:
if actual_results[1] in expected_results:
flakes[test] = actual_results[0]
else:
# We report the first failure type back, even if the second
# was more severe.
failures[test] = actual_results[0]
elif actual_results[0] == 'PASS':
passes[test] = result
else:
failures[test] = actual_results[0]
return (passes, failures, flakes)
def convert_trie_to_flat_paths(trie, prefix=None):
# Cloned from blinkpy.web_tests.layout_package.json_results_generator
# so that this code can stand alone.
result = {}
for name, data in trie.iteritems():
if prefix:
name = prefix + "/" + name
if len(data) and not "actual" in data and not "expected" in data:
result.update(convert_trie_to_flat_paths(data, name))
else:
result[name] = data
return result
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
|
