diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py')
| -rwxr-xr-x | Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py | 225 |
1 files changed, 176 insertions, 49 deletions
diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py index 389201521..ef459cd69 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py @@ -87,21 +87,48 @@ Ignoring warm-up run (1502) 1475 1471 -avg 1489.05 -median 1487 -stdev 14.46 -min 1471 -max 1510 +Time: +avg 1489.05 ms +median 1487 ms +stdev 14.46 ms +min 1471 ms +max 1510 ms """ elif driver_input.test_name.endswith('some-parser.html'): text = """Running 20 times Ignoring warm-up run (1115) -avg 1100 -median 1101 -stdev 11 -min 1080 -max 1120 +Time: +avg 1100 ms +median 1101 ms +stdev 11 ms +min 1080 ms +max 1120 ms +""" + elif driver_input.test_name.endswith('memory-test.html'): + text = """Running 20 times +Ignoring warm-up run (1115) + +Time: +avg 1100 ms +median 1101 ms +stdev 11 ms +min 1080 ms +max 1120 ms + +JS Heap: +avg 832000 bytes +median 829000 bytes +stdev 15000 bytes +min 811000 bytes +max 848000 bytes + +FastMalloc: +avg 532000 bytes +median 529000 bytes +stdev 13000 bytes +min 511000 bytes +max 548000 bytes """ return DriverOutput(text, '', '', '', crash=crash, timeout=timeout) @@ -120,6 +147,9 @@ max 1120 runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector') runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings') runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser') + + filesystem = runner._host.filesystem + runner.load_output_json = lambda: json.loads(filesystem.read_text_file(runner._output_json_path())) return runner, test_port def run_test(self, test_name): @@ -225,6 +255,33 @@ max 1120 'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms', '', ''])) + def test_run_memory_test(self): + runner, port = self.create_runner_and_setup_results_template() + runner._timestamp = 123456789 + port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content') + + output = OutputCapture() + output.capture_output() + try: + unexpected_result_count = runner.run() + finally: + stdout, stderr, log = output.restore_output() + self.assertEqual(unexpected_result_count, 0) + self.assertEqual(log, '\n'.join([ + 'Running 1 tests', + 'Running Parser/memory-test.html (1 of 1)', + 'RESULT Parser: memory-test= 1100.0 ms', + 'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms', + 'RESULT Parser: memory-test: JSHeap= 832000.0 bytes', + 'median= 829000.0 bytes, stdev= 15000.0 bytes, min= 811000.0 bytes, max= 848000.0 bytes', + 'RESULT Parser: memory-test: FastMalloc= 532000.0 bytes', + 'median= 529000.0 bytes, stdev= 13000.0 bytes, min= 511000.0 bytes, max= 548000.0 bytes', + '', ''])) + results = runner.load_output_json()[0]['results'] + self.assertEqual(results['Parser/memory-test'], {'min': 1080.0, 'max': 1120.0, 'median': 1101.0, 'stdev': 11.0, 'avg': 1100.0, 'unit': 'ms'}) + self.assertEqual(results['Parser/memory-test:JSHeap'], {'min': 811000.0, 'max': 848000.0, 'median': 829000.0, 'stdev': 15000.0, 'avg': 832000.0, 'unit': 'bytes'}) + self.assertEqual(results['Parser/memory-test:FastMalloc'], {'min': 511000.0, 'max': 548000.0, 'median': 529000.0, 'stdev': 13000.0, 'avg': 532000.0, 'unit': 'bytes'}) + def _test_run_with_json_output(self, runner, filesystem, upload_suceeds=True, expected_exit_code=0): filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') @@ -246,78 +303,148 @@ max 1120 finally: stdout, stderr, logs = output_capture.restore_output() - self.assertEqual(logs, '\n'.join([ - 'Running 2 tests', - 'Running Bindings/event-target-wrapper.html (1 of 2)', - 'RESULT Bindings: event-target-wrapper= 1489.05 ms', - 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms', - '', - 'Running inspector/pass.html (2 of 2)', - 'RESULT group_name: test_name= 42 ms', - '', - ''])) + if not expected_exit_code: + self.assertEqual(logs, '\n'.join([ + 'Running 2 tests', + 'Running Bindings/event-target-wrapper.html (1 of 2)', + 'RESULT Bindings: event-target-wrapper= 1489.05 ms', + 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms', + '', + 'Running inspector/pass.html (2 of 2)', + 'RESULT group_name: test_name= 42 ms', + '', + ''])) return uploaded[0] + _event_target_wrapper_and_inspector_results = { + "Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, + "inspector/pass.html:group_name:test_name": 42} + def test_run_with_json_output(self): runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', '--test-results-server=some.host']) self._test_run_with_json_output(runner, port.host.filesystem) - self.assertEqual(json.loads(port.host.filesystem.read_text_file('/mock-checkout/output.json')), { - "timestamp": 123456789, "results": - {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, - "inspector/pass.html:group_name:test_name": 42}, - "webkit-revision": 5678, "branch": "webkit-trunk"}) - - def test_run_generates_results_page(self): - runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json']) + self.assertEqual(runner.load_output_json(), { + "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "branch": "webkit-trunk"}) + + def test_run_with_description(self): + runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', + '--test-results-server=some.host', '--description', 'some description']) + self._test_run_with_json_output(runner, port.host.filesystem) + self.assertEqual(runner.load_output_json(), { + "timestamp": 123456789, "description": "some description", + "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "branch": "webkit-trunk"}) + + def create_runner_and_setup_results_template(self, args=[]): + runner, port = self.create_runner(args) filesystem = port.host.filesystem - print runner._base_path + '/resources/results-template.html' filesystem.write_text_file(runner._base_path + '/resources/results-template.html', - 'BEGIN<?WebKitPerfTestRunnerInsertionPoint?>END') - filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/dromaeo/web/lib/jquery-1.6.4.js', - 'jquery content') + 'BEGIN<script src="%AbsolutePathToWebKitTrunk%/some.js"></script>' + '<script src="%AbsolutePathToWebKitTrunk%/other.js"></script><script>%PeformanceTestsResultsJSON%</script>END') + filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/dromaeo/web/lib/jquery-1.6.4.js', 'jquery content') + return runner, port + def test_run_respects_no_results(self): + runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', + '--test-results-server=some.host', '--no-results']) + self.assertFalse(self._test_run_with_json_output(runner, port.host.filesystem)) + self.assertFalse(port.host.filesystem.isfile('/mock-checkout/output.json')) + + def test_run_generates_json_by_default(self): + runner, port = self.create_runner_and_setup_results_template() + filesystem = port.host.filesystem + output_json_path = filesystem.join(port.perf_results_directory(), runner._DEFAULT_JSON_FILENAME) + results_page_path = filesystem.splitext(output_json_path)[0] + '.html' + + self.assertFalse(filesystem.isfile(output_json_path)) + self.assertFalse(filesystem.isfile(results_page_path)) + + self._test_run_with_json_output(runner, port.host.filesystem) + + self.assertEqual(json.loads(port.host.filesystem.read_text_file(output_json_path)), [{ + "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "branch": "webkit-trunk"}]) + + self.assertTrue(filesystem.isfile(output_json_path)) + self.assertTrue(filesystem.isfile(results_page_path)) + + def test_run_generates_and_show_results_page(self): + runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) + page_shown = [] + port.show_results_html_file = lambda path: page_shown.append(path) + filesystem = port.host.filesystem self._test_run_with_json_output(runner, filesystem) - expected_entry = {"timestamp": 123456789, "results": {"Bindings/event-target-wrapper": - {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, - "inspector/pass.html:group_name:test_name": 42}, "webkit-revision": 5678} + expected_entry = {"timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "branch": "webkit-trunk"} self.maxDiff = None json_output = port.host.filesystem.read_text_file('/mock-checkout/output.json') self.assertEqual(json.loads(json_output), [expected_entry]) self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), - 'BEGIN<script>jquery content</script><script id="json">' + json_output + '</script>END') + 'BEGIN<script src="/test.checkout/some.js"></script><script src="/test.checkout/other.js"></script>' + '<script>%s</script>END' % json_output) + self.assertEqual(page_shown[0], '/mock-checkout/output.html') self._test_run_with_json_output(runner, filesystem) json_output = port.host.filesystem.read_text_file('/mock-checkout/output.json') self.assertEqual(json.loads(json_output), [expected_entry, expected_entry]) self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), - 'BEGIN<script>jquery content</script><script id="json">' + json_output + '</script>END') + 'BEGIN<script src="/test.checkout/some.js"></script><script src="/test.checkout/other.js"></script>' + '<script>%s</script>END' % json_output) + + def test_run_respects_no_show_results(self): + show_results_html_file = lambda path: page_shown.append(path) + + runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) + page_shown = [] + port.show_results_html_file = show_results_html_file + self._test_run_with_json_output(runner, port.host.filesystem) + self.assertEqual(page_shown[0], '/mock-checkout/output.html') + + runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', + '--no-show-results']) + page_shown = [] + port.show_results_html_file = show_results_html_file + self._test_run_with_json_output(runner, port.host.filesystem) + self.assertEqual(page_shown, []) + + def test_run_with_bad_output_json(self): + runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) + port.host.filesystem.write_text_file('/mock-checkout/output.json', 'bad json') + self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) + port.host.filesystem.write_text_file('/mock-checkout/output.json', '{"another bad json": "1"}') + self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) def test_run_with_json_source(self): runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', '--source-json-path=/mock-checkout/source.json', '--test-results-server=some.host']) port.host.filesystem.write_text_file('/mock-checkout/source.json', '{"key": "value"}') self._test_run_with_json_output(runner, port.host.filesystem) - self.assertEqual(json.loads(port.host.filesystem.files['/mock-checkout/output.json']), { - "timestamp": 123456789, "results": - {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, - "inspector/pass.html:group_name:test_name": 42}, - "webkit-revision": 5678, "branch": "webkit-trunk", - "key": "value"}) + self.assertEqual(runner.load_output_json(), { + "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "branch": "webkit-trunk", "key": "value"}) + + def test_run_with_bad_json_source(self): + runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', + '--source-json-path=/mock-checkout/source.json', '--test-results-server=some.host']) + self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) + port.host.filesystem.write_text_file('/mock-checkout/source.json', 'bad json') + self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) + port.host.filesystem.write_text_file('/mock-checkout/source.json', '["another bad json"]') + self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) def test_run_with_multiple_repositories(self): runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', '--test-results-server=some.host']) port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')] self._test_run_with_json_output(runner, port.host.filesystem) - self.assertEqual(json.loads(port.host.filesystem.files['/mock-checkout/output.json']), { - "timestamp": 123456789, "results": - {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, - "inspector/pass.html:group_name:test_name": 42.0}, - "webkit-revision": 5678, "some-revision": 5678, "branch": "webkit-trunk"}) + self.assertEqual(runner.load_output_json(), { + "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results, + "webkit-revision": "5678", "some-revision": "5678", "branch": "webkit-trunk"}) def test_run_with_upload_json(self): runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', @@ -329,7 +456,7 @@ max 1120 self.assertEqual(generated_json['builder-name'], 'builder1') self.assertEqual(generated_json['build-number'], 123) - self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=False, expected_exit_code=-3) + self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=False, expected_exit_code=PerfTestsRunner.EXIT_CODE_FAILED_UPLOADING) def test_upload_json(self): runner, port = self.create_runner() |