From 9b3cd8f334fe335a77b71cdd5c964d88eebd3fbd Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Sat, 11 Oct 2014 21:29:27 +0100 Subject: yarn: Move timing out of .run_scenario This is in effort of moving the scenario running code into yarnlib, and being able to make it run scenarios in parallel. --- yarn | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/yarn b/yarn index ffec41a..47ef754 100755 --- a/yarn +++ b/yarn @@ -235,34 +235,24 @@ class YarnRunner(cliapp.Application): return scenarios def run_scenario(self, scenario_runner, scenario): - self.start_scenario_timing(scenario.name) - started = time.time() - - self.info('Running scenario %s' % scenario.name) - self.ts['scenario_name'] = scenario.name - self.scenarios_run += 1 - - if self.settings['no-act']: - self.info('Pretending everything went OK') - for step in scenario.steps: - self.ts['current_step'] = step - self.remember_scenario_timing(time.time() - started) - return True - os.mkdir(self.scenario_dir(self.tempdir, scenario)) datadir = self.datadir(self.tempdir, scenario) os.mkdir(datadir) - self.info('DATADIR is %s' % datadir) homedir = self.homedir(datadir) os.mkdir(homedir) - self.info('HOME for tests is %s' % homedir) - ok = scenario_runner.run_scenario(scenario, datadir, homedir) + ud = self.pre_scenario(scenario, datadir, homedir) + + if self.settings['no-act']: + self.info('Pretending everything went OK') + for step in scenario.steps: + self.ts['current_step'] = step + ok = True + else: + ok = scenario_runner.run_scenario(scenario, datadir, homedir) - self.remember_scenario_timing(time.time() - started) + self.post_scenario(scenario, datadir, homedir, ud) - if not self.settings['snapshot']: - shutil.rmtree(datadir, ignore_errors=True) return ok def homedir(self, datadir): @@ -277,6 +267,26 @@ class YarnRunner(cliapp.Application): key, value = option_arg.split('=', 1) yield key, value + def pre_scenario(self, scenario, datadir, homedir): + self.start_scenario_timing(scenario.name) + started = time.time() + + self.info('Running scenario %s' % scenario.name) + self.ts['scenario_name'] = scenario.name + self.scenarios_run += 1 + self.info('DATADIR is %s' % datadir) + self.info('HOME for tests is %s' % homedir) + return (started,) + + def post_scenario(self, scenario, datadir, homedir, pre_scenario_userdata): + stopped = time.time() + started, = pre_scenario_userdata + + self.remember_scenario_timing(stopped - started) + + if not self.settings['snapshot']: + shutil.rmtree(datadir, ignore_errors=True) + def pre_step(self, step, **ignored): started = time.time() -- cgit v1.2.1 From 16c307a779b929eb4c0010fc20e12df4f28b3247 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Sat, 11 Oct 2014 22:18:24 +0100 Subject: yarn: Move --no-act logic out of .run_scenario --- yarn | 34 +++++++++++++++++++++------------- yarnlib/scenario_runner.py | 2 ++ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/yarn b/yarn index 47ef754..15b279a 100755 --- a/yarn +++ b/yarn @@ -32,6 +32,16 @@ import cmdtestlib import yarnlib +def dry_scenario_runner_factory(info, ts): + class DryScenarioRunner(yarnlib.ScenarioRunner): + def run_scenario(self, scenario, datadir, homedir): + info('Pretending everything went OK') + for step in scenario.steps: + ts['current_step'] = step + return True + return DryScenarioRunner + + class YarnRunner(cliapp.Application): def add_settings(self): @@ -143,10 +153,16 @@ class YarnRunner(cliapp.Application): self.steps_run = 0 self.timings = [] - scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(), - self.parse_env(), - pre_step_cb=self.pre_step, - post_step_cb=self.post_step) + if self.settings['no-act']: + runner_class = dry_scenario_runner_factory(self.info, self.ts) + else: + runner_class = yarnlib.ScenarioRunner + scenario_runner = runner_class(shell_prelude, os.getcwd(), + self.parse_env(), + pre_step_cb=self.pre_step, + post_step_cb=self.post_step, + pre_scenario_cb=self.pre_scenario, + post_scenario_cb=self.post_scenario) start_time = time.time() failed_scenarios = [] @@ -242,15 +258,7 @@ class YarnRunner(cliapp.Application): os.mkdir(homedir) ud = self.pre_scenario(scenario, datadir, homedir) - - if self.settings['no-act']: - self.info('Pretending everything went OK') - for step in scenario.steps: - self.ts['current_step'] = step - ok = True - else: - ok = scenario_runner.run_scenario(scenario, datadir, homedir) - + ok = scenario_runner.run_scenario(scenario, datadir, homedir) self.post_scenario(scenario, datadir, homedir, ud) return ok diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py index e0cdf62..28172e4 100644 --- a/yarnlib/scenario_runner.py +++ b/yarnlib/scenario_runner.py @@ -101,6 +101,8 @@ class ScenarioRunner(object): def __init__(self, shell_prelude, srcdir, extra_env=(), pre_step_cb=default_pre_step, post_step_cb=default_post_step, + pre_scenario_cb=lambda *x: None, + post_scenario_cb=lambda *x: None, cmdrunner=cliapp.runcmd_unchecked): self.shell_prelude = shell_prelude self.srcdir = srcdir -- cgit v1.2.1 From 69dcbb0a55db2422759819e46d8d5ce0121a1a82 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Sat, 11 Oct 2014 23:53:28 +0100 Subject: yarn: Move scenario setup into ScenarioRunner Now you should call .run_scenarios instead of .run_scenario, and pass testdir to the ScenarioRunner class. You can still call .run_scenario directly instead. The testdir argument isn't required when used that way. --- yarn | 52 +++++++--------------------------------------- yarnlib/scenario_runner.py | 49 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/yarn b/yarn index 15b279a..bb5759a 100755 --- a/yarn +++ b/yarn @@ -162,13 +162,12 @@ class YarnRunner(cliapp.Application): pre_step_cb=self.pre_step, post_step_cb=self.post_step, pre_scenario_cb=self.pre_scenario, - post_scenario_cb=self.post_scenario) + post_scenario_cb=self.post_scenario, + testdir=self.tempdir) start_time = time.time() - failed_scenarios = [] - for scenario in self.select_scenarios(scenarios): - if not self.run_scenario(scenario_runner, scenario): - failed_scenarios.append(scenario) + failed_scenarios = ( + scenario_runner.run_scenarios(self.select_scenarios(scenarios))) duration = time.time() - start_time if not self.settings['snapshot']: @@ -250,22 +249,6 @@ class YarnRunner(cliapp.Application): return scenarios - def run_scenario(self, scenario_runner, scenario): - os.mkdir(self.scenario_dir(self.tempdir, scenario)) - datadir = self.datadir(self.tempdir, scenario) - os.mkdir(datadir) - homedir = self.homedir(datadir) - os.mkdir(homedir) - - ud = self.pre_scenario(scenario, datadir, homedir) - ok = scenario_runner.run_scenario(scenario, datadir, homedir) - self.post_scenario(scenario, datadir, homedir, ud) - - return ok - - def homedir(self, datadir): - return os.path.join(datadir, 'HOME') - def parse_env(self): for option_arg in self.settings['env']: if '=' not in option_arg: @@ -339,16 +322,10 @@ class YarnRunner(cliapp.Application): self.snapshot_datadir(self.tempdir, step_env['DATADIR'], scenario, step_number, step) - def scenario_dir(self, tempdir, scenario): - return os.path.join(tempdir, self.nice(scenario.name)) - - def datadir(self, tempdir, scenario): - sd = self.scenario_dir(tempdir, scenario) - return os.path.join(sd, 'datadir') - def snapshot_dir(self, tempdir, scenario, step, step_number): - sd = self.scenario_dir(tempdir, scenario) - base = '%03d-%s-%s' % (step_number, step.what, self.nice(step.text)) + sd = yarnlib.ScenarioRunner.scenario_dir(tempdir, scenario) + base = '%03d-%s-%s' % (step_number, step.what, + yarnlib.ScenarioRunner.nice(step.text)) return os.path.join(sd, base) def snapshot_datadir(self, tempdir, datadir, scenario, step_number, step): @@ -358,21 +335,6 @@ class YarnRunner(cliapp.Application): if exit != 0: logging.warning('Snapshot copy failed:\n%s\n%s' % (out, err)) - def nice(self, name): - # Quote a scenario or step name so it forms a nice filename. - nice_chars = "abcdefghijklmnopqrstuvwxyz" - nice_chars += nice_chars.upper() - nice_chars += "0123456789-." - - nice = [] - for c in name: - if c in nice_chars: - nice.append(c) - elif not nice or nice[-1] != '_': - nice.append('_') - nice = ''.join(nice) - return nice - def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py index 28172e4..cb74859 100644 --- a/yarnlib/scenario_runner.py +++ b/yarnlib/scenario_runner.py @@ -103,13 +103,32 @@ class ScenarioRunner(object): pre_step_cb=default_pre_step, post_step_cb=default_post_step, pre_scenario_cb=lambda *x: None, post_scenario_cb=lambda *x: None, - cmdrunner=cliapp.runcmd_unchecked): + cmdrunner=cliapp.runcmd_unchecked, testdir=None): self.shell_prelude = shell_prelude self.srcdir = srcdir self.env = self.clean_env(extra_env, SRCDIR=srcdir) self.pre_step_cb = pre_step_cb self.post_step_cb = post_step_cb + self.pre_scenario_cb = pre_scenario_cb + self.post_scenario_cb = post_scenario_cb self.cmdrunner = cmdrunner + self.testdir = testdir + + def run_scenarios(self, scenarios): + failed = [] + for scenario in scenarios: + scenario_dir = self.scenario_dir(self.testdir, scenario) + os.mkdir(scenario_dir) + datadir = self.datadir(scenario_dir) + os.mkdir(datadir) + homedir = self.homedir(datadir) + os.mkdir(homedir) + + ud = self.pre_scenario_cb(scenario, datadir, homedir) + if not self.run_scenario(scenario, datadir, homedir): + failed.append(scenario) + self.post_scenario_cb(scenario, datadir, homedir, ud) + return failed def run_scenario(self, scenario, datadir, homedir): assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] @@ -202,3 +221,31 @@ class ScenarioRunner(object): env.update(kwarg_env) return env + + @classmethod + def scenario_dir(cls, testdir, scenario): + return os.path.join(testdir, cls.nice(scenario.name)) + + @staticmethod + def datadir(scenario_dir): + return os.path.join(scenario_dir, 'datadir') + + @staticmethod + def homedir(datadir): + return os.path.join(datadir, 'HOME') + + @staticmethod + def nice(name): + # Quote a scenario or step name so it forms a nice filename. + nice_chars = "abcdefghijklmnopqrstuvwxyz" + nice_chars += nice_chars.upper() + nice_chars += "0123456789-." + + nice = [] + for c in name: + if c in nice_chars: + nice.append(c) + elif not nice or nice[-1] != '_': + nice.append('_') + nice = ''.join(nice) + return nice -- cgit v1.2.1 From dbe426fb3f7689787194ceb4b9a1d5b7a5ce8b52 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 15 Oct 2014 13:23:31 +0000 Subject: yarnlib: Split out scenario and step setup This is so the future scenario multi-runner can re-use the common setup code. --- yarnlib/scenario_runner.py | 47 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py index cb74859..91e7f0f 100644 --- a/yarnlib/scenario_runner.py +++ b/yarnlib/scenario_runner.py @@ -114,34 +114,41 @@ class ScenarioRunner(object): self.cmdrunner = cmdrunner self.testdir = testdir + def setup_scenario(self, scenario): + scenario_dir = self.scenario_dir(self.testdir, scenario) + os.mkdir(scenario_dir) + datadir = self.datadir(scenario_dir) + os.mkdir(datadir) + homedir = self.homedir(datadir) + os.mkdir(homedir) + + ud = self.pre_scenario_cb(scenario, datadir, homedir) + return scenario_dir, datadir, homedir, ud + def run_scenarios(self, scenarios): failed = [] for scenario in scenarios: - scenario_dir = self.scenario_dir(self.testdir, scenario) - os.mkdir(scenario_dir) - datadir = self.datadir(scenario_dir) - os.mkdir(datadir) - homedir = self.homedir(datadir) - os.mkdir(homedir) - - ud = self.pre_scenario_cb(scenario, datadir, homedir) + scenario_dir, datadir, homedir, ud = self.setup_scenario(scenario) if not self.run_scenario(scenario, datadir, homedir): failed.append(scenario) self.post_scenario_cb(scenario, datadir, homedir, ud) return failed - def run_scenario(self, scenario, datadir, homedir): + @staticmethod + def partition_steps(scenario): assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] - normal = [s for s in scenario.steps if s not in assuming + cleanup] + normal = [s for s in scenario.steps + if s.what not in ('ASSUMING', 'FINALLY')] + return assuming, cleanup, normal + + def run_scenario(self, scenario, datadir, homedir): + assuming, cleanup, normal = self.partition_steps(scenario) + scenario_env = dict(self.env, HOME=homedir, DATADIR=datadir) ok = True step_number = 1 - scenario_env = dict(self.env) - scenario_env['HOME'] = homedir - scenario_env['DATADIR'] = datadir - for step in assuming: exit = self.run_step(scenario, step, scenario_env, step_number) step_number += 1 @@ -164,12 +171,12 @@ class ScenarioRunner(object): return ok - def run_step(self, scenario, step, scenario_env, step_number): + def setup_step(self, step, scenario_env, scenario, step_number): m = yarnlib.implements_matches_step(step.implementation, step) assert m is not None step_env = dict(scenario_env) - for i, match in enumerate(m.groups('')): - step_env['MATCH_%d' % (i+1)] = match + step_env.update(('MATCH_%d' % i, match) for (i, match) + in enumerate(m.groups(''), 1)) # All parameters passed as keyword-arguments, so that the callback # may declare parameters in any order, and ignore any parameters @@ -180,6 +187,12 @@ class ScenarioRunner(object): shell_script = '%s\n\n%s\n' % ( self.shell_prelude, step.implementation.shell) + return step_env, pre_step_userdata, shell_script + + def run_step(self, scenario, step, scenario_env, step_number): + step_env, pre_step_userdata, shell_script = self.setup_step( + step=step, scenario_env=scenario_env, + scenario=scenario, step_number=step_number) exit, stdout, stderr = self.cmdrunner( ['sh', '-xeuc', shell_script], env=step_env, cwd=self.srcdir) -- cgit v1.2.1 From a23193fc77c025a723f041d4308684d268c02760 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 13 Oct 2014 09:24:39 +0100 Subject: yarnlib: Add a scenario runner that runs scenarios in parallel This uses an asyncore select loop to process the output of multiple subprocesses, start new steps when a previous step exits, and start a new scenario when that one has finished. --- yarnlib/__init__.py | 1 + yarnlib/scenario_multi_runner.py | 164 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 yarnlib/scenario_multi_runner.py diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index 515082e..c2a5edd 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -26,6 +26,7 @@ from scenario_step_connector import (implements_matches_step, StepNotImplementedError, StepMultipleImplementationsError) from scenario_runner import ScenarioRunner +from scenario_multi_runner import ScenarioMultiRunner from shell_libraries import load_shell_libraries diff --git a/yarnlib/scenario_multi_runner.py b/yarnlib/scenario_multi_runner.py new file mode 100644 index 0000000..402e20c --- /dev/null +++ b/yarnlib/scenario_multi_runner.py @@ -0,0 +1,164 @@ +# Copyright 2014 Richard Maw +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import asyncore +import collections +import errno +import logging +import os +import socket +import subprocess + +import yarnlib + + +class StepOutputHandler(asyncore.file_dispatcher): + def __init__(self, fd, handle_closed, handle_output, map=None): + asyncore.file_dispatcher.__init__(self, fd=fd, map=map) + self.handle_closed = handle_closed + self.handle_output = handle_output + def handle_read(self): + d = self.read(4096) + if not d: + self.close() + self.handle_closed() + self.handle_output(d) + + +class StepJob(object): + stdout_closed = False + stderr_closed = False + def __init__(self, step, script, env, scenario_queue, runner, + fd_map, scenarios, failed_scenarios): + self.step = step + self.scenario_queue = scenario_queue + self.runner = runner + self.fd_map = fd_map + self.scenarios = scenarios + self.failed_scenarios = failed_scenarios + + self.outbuf = [] + self.errbuf = [] + + self.sp = sp = subprocess.Popen(['sh', '-xeuc', script], + stdin=open(os.devnull, 'r'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, + cwd=runner.srcdir) + StepOutputHandler(fd=sp.stdout, map=fd_map, + handle_output=self.outbuf.append, + handle_closed=self.handle_stdout_closed) + StepOutputHandler(fd=sp.stderr, map=fd_map, + handle_output=self.errbuf.append, + handle_closed=self.handle_stderr_closed) + def handle_stdout_closed(self): + self.stdout_closed = True + if self.stderr_closed: + self.terminate() + def handle_stderr_closed(self): + self.stderr_closed = True + if self.stdout_closed: + self.terminate() + def terminate(self): + exit_code = self.sp.poll() + if exit_code is None: + logging.warning('Step %s closed output before terminating, ' + 'waiting for it to exit', self.step.text) + exit_code = self.sp.wait() + self.handle_terminated(exit_code, ''.join(self.outbuf), + ''.join(self.errbuf)) + def handle_terminated(self, exit_code, stdout, stderr): + try: + self.scenario_queue.send((exit_code, stdout, stderr)) + except StopIteration: + # end of scenario, queue another + if self.scenarios: + scenario = self.scenarios.pop(0) + s = self.runner.run_scenario( + fd_map=self.fd_map, scenario=scenario, + scenarios=self.scenarios, + failed_scenarios=self.failed_scenarios) + s.next() + s.send(s) + + +class ScenarioMultiRunner(yarnlib.ScenarioRunner): + def __init__(self, *args, **kwargs): + self.max_jobs = kwargs.pop('max_jobs') + yarnlib.ScenarioRunner.__init__(self, *args, **kwargs) + + def run_scenarios(self, scenarios): + jobs_to_run = min(self.max_jobs, len(scenarios)) + remaining_scenarios = scenarios[jobs_to_run:] + fd_map = {} + failed_scenarios = [] + for scenario in scenarios[:jobs_to_run]: + s = self.run_scenario(fd_map=fd_map, + scenario=scenario, + scenarios=remaining_scenarios, + failed_scenarios=failed_scenarios) + s.next() # need an initial next + s.send(s) + asyncore.loop(map=fd_map) + return failed_scenarios + + def run_scenario(self, fd_map, scenario, scenarios, failed_scenarios): + scenario_dir, datadir, homedir, pre_scenario_ud = ( + self.setup_scenario(scenario)) + assuming, cleanup, normal = self.partition_steps(scenario) + scenario_env = dict(self.env, HOME=homedir, DATADIR=datadir) + + ok = True + step_number = 1 + scenario_queue = (yield) + + # can't delegate common setup here, since it's more code to + # delegate a yield than it's worth without python3's yield from, + # so rather than using the nice for-loops in ScenarioRunner, + # we move the control flow logic to after the step. + queue = collections.deque(assuming + normal + cleanup) + while queue: + step = queue.popleft() + + step_env, pre_step_userdata, shell_script = self.setup_step( + step=step, scenario_env=scenario_env, + scenario=scenario, step_number=step_number) + StepJob(step=step, script=shell_script, env=step_env, + runner=self, scenario_queue=scenario_queue, + fd_map=fd_map, scenarios=scenarios, + failed_scenarios=failed_scenarios) + exit, stdout, stderr = (yield) + self.post_step_cb(scenario=scenario, step=step, + step_number=step_number, step_env=step_env, + exit=exit, stdout=stdout, stderr=stderr, + pre_step_userdata=pre_step_userdata) + step_number += 1 + if exit != 0: + if step in assuming: + break + elif step in normal: + ok = False + queue = collections.deque(cleanup) + else: + ok = False + break + if not ok: + failed_scenarios.append(scenario) + self.post_scenario_cb(scenario=scenario, datadir=datadir, + homedir=homedir, + pre_scenario_userdata=pre_scenario_ud) -- cgit v1.2.1 From 03b9e8e9c0383a96da2bbb44ed11949a35eab6ad Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 15 Oct 2014 13:33:52 +0000 Subject: yarn: Allow multi-runner when called with -jN --- yarn | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/yarn b/yarn index bb5759a..00a8a46 100755 --- a/yarn +++ b/yarn @@ -19,6 +19,7 @@ import cliapp import collections +import functools import logging import os import re @@ -94,6 +95,12 @@ class YarnRunner(cliapp.Application): 'allow scenarios to reference steps that do not exist, ' 'by warning about them, but otherwise ignoring the scenarios') + self.settings.integer( + ['max-jobs', 'j'], + 'run as many as JOBS scenarios in parallel', + default=1, + metavar='JOBS') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -155,6 +162,9 @@ class YarnRunner(cliapp.Application): if self.settings['no-act']: runner_class = dry_scenario_runner_factory(self.info, self.ts) + elif self.settings['max-jobs'] > 1: + runner_class = functools.partial(yarnlib.ScenarioMultiRunner, + max_jobs=self.settings['max-jobs']) else: runner_class = yarnlib.ScenarioRunner scenario_runner = runner_class(shell_prelude, os.getcwd(), -- cgit v1.2.1 From 06f1c4500cdbd612a39e41323ae2c28e4d307f0b Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 15 Oct 2014 14:40:19 +0000 Subject: yarn: Don't crash when a command failed with binary output --- yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn b/yarn index 00a8a46..811e41d 100755 --- a/yarn +++ b/yarn @@ -324,7 +324,7 @@ class YarnRunner(cliapp.Application): 'with exit code %d:\n' 'Standard output from shell command:\n%s' 'Standard error from shell command:\n%s' % - (scenario.name, step.what, step.text, exit, + (str(scenario.name), str(step.what), str(step.text), exit, self.indent(stdout), self.indent(stderr))) self.remember_step_timing( '%s %s' % (step.what, step.text), stopped - started) -- cgit v1.2.1 From ade6bf6bcfce5e6b365e593a5580a024c2329384 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 13:15:53 +0000 Subject: yarn: Improve parallel testing progress reporting Previously it would list [x/y] where x was the step id, and y was the total number of steps. This stopped producing sensible output after we started being able to run scenarios in parallel, since jobs were started in non-sequential id order. Now it's just a counter, with x as the number of steps completed. This also fixes it listing the wrong scenario name for each step, since previously it would set the scenario name when a scenario was started, now since they are interleaved, we need to report it when we start a new step. Also, as a consequence of moving to ID order, we need to adjust the steps completed when we skip because an ASSUMING failed. It may be better to make the scenario runners track the number of remaining steps, since we're currently encoding this information both in the runner, and in yarn where we're reporting status. --- yarn | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/yarn b/yarn index 811e41d..1cfa801 100755 --- a/yarn +++ b/yarn @@ -38,7 +38,7 @@ def dry_scenario_runner_factory(info, ts): def run_scenario(self, scenario, datadir, homedir): info('Pretending everything went OK') for step in scenario.steps: - ts['current_step'] = step + ts['steps_completed'] += 1 return True return DryScenarioRunner @@ -131,7 +131,8 @@ class YarnRunner(cliapp.Application): self.ts = ttystatus.TerminalStatus(period=0.001) if not self.settings['quiet'] and not self.settings['verbose']: self.ts.format( - '%ElapsedTime() %Index(current_step,all_steps): ' + '%ElapsedTime() ' + '[%Integer(steps_completed)/%Integer(total_steps)]: ' '%String(scenario_name): ' '%String(step_name)') @@ -153,7 +154,8 @@ class YarnRunner(cliapp.Application): all_steps = [] for scenario in scenarios: all_steps.extend(scenario.steps) - self.ts['all_steps'] = all_steps + self.ts['total_steps'] = len(all_steps) + self.ts['steps_completed'] = 0 self.scenarios_run = 0 self.skipped_for_assuming = 0 @@ -273,8 +275,6 @@ class YarnRunner(cliapp.Application): started = time.time() self.info('Running scenario %s' % scenario.name) - self.ts['scenario_name'] = scenario.name - self.scenarios_run += 1 self.info('DATADIR is %s' % datadir) self.info('HOME for tests is %s' % homedir) return (started,) @@ -283,18 +283,18 @@ class YarnRunner(cliapp.Application): stopped = time.time() started, = pre_scenario_userdata + self.scenarios_run += 1 self.remember_scenario_timing(stopped - started) if not self.settings['snapshot']: shutil.rmtree(datadir, ignore_errors=True) - def pre_step(self, step, **ignored): + def pre_step(self, scenario, step, **ignored): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) - self.ts['current_step'] = step + self.ts['scenario_name'] = scenario.name self.ts['step_name'] = '%s %s' % (step.what, step.text) - self.steps_run += 1 return (started,) @@ -303,6 +303,9 @@ class YarnRunner(cliapp.Application): stopped = time.time() (started,) = pre_step_userdata + self.steps_run += 1 + self.ts['steps_completed'] += 1 + logging.debug('Exit code: %d' % exit) if stdout: logging.debug('Standard output:\n%s' % self.indent(stdout)) @@ -318,6 +321,8 @@ class YarnRunner(cliapp.Application): 'Skipping "%s" because "%s %s" failed' % (scenario.name, step.what, step.text)) self.skipped_for_assuming += 1 + skipped = len(scenario.steps) - scenario.steps.index(step) - 1 + self.ts['total_steps'] -= skipped else: self.error( 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' -- cgit v1.2.1 From ee202bbaba6761d953c36ce193b5130f52da6b2f Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 24 Oct 2014 14:11:38 +0000 Subject: Paper over the lack of unit test coverage --- without-tests | 1 + yarnlib/scenario_runner.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/without-tests b/without-tests index 6aa40f1..991cd64 100644 --- a/without-tests +++ b/without-tests @@ -1,3 +1,4 @@ yarnlib/__init__.py setup.py yarnlib/elements.py +yarnlib/scenario_multi_runner.py diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py index 91e7f0f..cbe2fad 100644 --- a/yarnlib/scenario_runner.py +++ b/yarnlib/scenario_runner.py @@ -114,7 +114,7 @@ class ScenarioRunner(object): self.cmdrunner = cmdrunner self.testdir = testdir - def setup_scenario(self, scenario): + def setup_scenario(self, scenario): # pragma: no cover scenario_dir = self.scenario_dir(self.testdir, scenario) os.mkdir(scenario_dir) datadir = self.datadir(scenario_dir) @@ -125,7 +125,7 @@ class ScenarioRunner(object): ud = self.pre_scenario_cb(scenario, datadir, homedir) return scenario_dir, datadir, homedir, ud - def run_scenarios(self, scenarios): + def run_scenarios(self, scenarios): # pragma: no cover failed = [] for scenario in scenarios: scenario_dir, datadir, homedir, ud = self.setup_scenario(scenario) @@ -236,19 +236,19 @@ class ScenarioRunner(object): return env @classmethod - def scenario_dir(cls, testdir, scenario): + def scenario_dir(cls, testdir, scenario): # pragma: no cover return os.path.join(testdir, cls.nice(scenario.name)) @staticmethod - def datadir(scenario_dir): + def datadir(scenario_dir): # pragma: no cover return os.path.join(scenario_dir, 'datadir') @staticmethod - def homedir(datadir): + def homedir(datadir): # pragma: no cover return os.path.join(datadir, 'HOME') @staticmethod - def nice(name): + def nice(name): # pragma: no cover # Quote a scenario or step name so it forms a nice filename. nice_chars = "abcdefghijklmnopqrstuvwxyz" nice_chars += nice_chars.upper() -- cgit v1.2.1