diff options
Diffstat (limited to 'yarn')
| -rwxr-xr-x | yarn | 133 |
1 files changed, 64 insertions, 69 deletions
@@ -19,6 +19,7 @@ import cliapp import collections +import functools import logging import os import re @@ -32,6 +33,16 @@ import cmdtestlib import yarnlib +def dry_scenario_runner_factory(info, ts): + class DryScenarioRunner(yarnlib.ScenarioRunner): + def run_scenario(self, scenario, datadir, homedir): + info('Pretending everything went OK') + for step in scenario.steps: + ts['steps_completed'] += 1 + return True + return DryScenarioRunner + + class YarnRunner(cliapp.Application): def add_settings(self): @@ -84,6 +95,12 @@ class YarnRunner(cliapp.Application): 'allow scenarios to reference steps that do not exist, ' 'by warning about them, but otherwise ignoring the scenarios') + self.settings.integer( + ['max-jobs', 'j'], + 'run as many as JOBS scenarios in parallel', + default=1, + metavar='JOBS') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -114,7 +131,8 @@ class YarnRunner(cliapp.Application): self.ts = ttystatus.TerminalStatus(period=0.001) if not self.settings['quiet'] and not self.settings['verbose']: self.ts.format( - '%ElapsedTime() %Index(current_step,all_steps): ' + '%ElapsedTime() ' + '[%Integer(steps_completed)/%Integer(total_steps)]: ' '%String(scenario_name): ' '%String(step_name)') @@ -136,23 +154,32 @@ class YarnRunner(cliapp.Application): all_steps = [] for scenario in scenarios: all_steps.extend(scenario.steps) - self.ts['all_steps'] = all_steps + self.ts['total_steps'] = len(all_steps) + self.ts['steps_completed'] = 0 self.scenarios_run = 0 self.skipped_for_assuming = 0 self.steps_run = 0 self.timings = [] - scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(), - self.parse_env(), - pre_step_cb=self.pre_step, - post_step_cb=self.post_step) + if self.settings['no-act']: + runner_class = dry_scenario_runner_factory(self.info, self.ts) + elif self.settings['max-jobs'] > 1: + runner_class = functools.partial(yarnlib.ScenarioMultiRunner, + max_jobs=self.settings['max-jobs']) + else: + runner_class = yarnlib.ScenarioRunner + scenario_runner = runner_class(shell_prelude, os.getcwd(), + self.parse_env(), + pre_step_cb=self.pre_step, + post_step_cb=self.post_step, + pre_scenario_cb=self.pre_scenario, + post_scenario_cb=self.post_scenario, + testdir=self.tempdir) start_time = time.time() - failed_scenarios = [] - for scenario in self.select_scenarios(scenarios): - if not self.run_scenario(scenario_runner, scenario): - failed_scenarios.append(scenario) + failed_scenarios = ( + scenario_runner.run_scenarios(self.select_scenarios(scenarios))) duration = time.time() - start_time if not self.settings['snapshot']: @@ -234,56 +261,40 @@ class YarnRunner(cliapp.Application): return scenarios - def run_scenario(self, scenario_runner, scenario): + def parse_env(self): + for option_arg in self.settings['env']: + if '=' not in option_arg: + raise cliapp.AppException( + '--env argument must contain "=" ' + 'to separate environment variable name and value') + key, value = option_arg.split('=', 1) + yield key, value + + def pre_scenario(self, scenario, datadir, homedir): self.start_scenario_timing(scenario.name) started = time.time() self.info('Running scenario %s' % scenario.name) - self.ts['scenario_name'] = scenario.name - self.scenarios_run += 1 - - if self.settings['no-act']: - self.info('Pretending everything went OK') - for step in scenario.steps: - self.ts['current_step'] = step - self.remember_scenario_timing(time.time() - started) - return True - - os.mkdir(self.scenario_dir(self.tempdir, scenario)) - datadir = self.datadir(self.tempdir, scenario) - os.mkdir(datadir) self.info('DATADIR is %s' % datadir) - homedir = self.homedir(datadir) - os.mkdir(homedir) self.info('HOME for tests is %s' % homedir) + return (started,) - ok = scenario_runner.run_scenario(scenario, datadir, homedir) + def post_scenario(self, scenario, datadir, homedir, pre_scenario_userdata): + stopped = time.time() + started, = pre_scenario_userdata - self.remember_scenario_timing(time.time() - started) + self.scenarios_run += 1 + self.remember_scenario_timing(stopped - started) if not self.settings['snapshot']: shutil.rmtree(datadir, ignore_errors=True) - return ok - - def homedir(self, datadir): - return os.path.join(datadir, 'HOME') - def parse_env(self): - for option_arg in self.settings['env']: - if '=' not in option_arg: - raise cliapp.AppException( - '--env argument must contain "=" ' - 'to separate environment variable name and value') - key, value = option_arg.split('=', 1) - yield key, value - - def pre_step(self, step, **ignored): + def pre_step(self, scenario, step, **ignored): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) - self.ts['current_step'] = step + self.ts['scenario_name'] = scenario.name self.ts['step_name'] = '%s %s' % (step.what, step.text) - self.steps_run += 1 return (started,) @@ -292,6 +303,9 @@ class YarnRunner(cliapp.Application): stopped = time.time() (started,) = pre_step_userdata + self.steps_run += 1 + self.ts['steps_completed'] += 1 + logging.debug('Exit code: %d' % exit) if stdout: logging.debug('Standard output:\n%s' % self.indent(stdout)) @@ -307,13 +321,15 @@ class YarnRunner(cliapp.Application): 'Skipping "%s" because "%s %s" failed' % (scenario.name, step.what, step.text)) self.skipped_for_assuming += 1 + skipped = len(scenario.steps) - scenario.steps.index(step) - 1 + self.ts['total_steps'] -= skipped else: self.error( 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' 'with exit code %d:\n' 'Standard output from shell command:\n%s' 'Standard error from shell command:\n%s' % - (scenario.name, step.what, step.text, exit, + (str(scenario.name), str(step.what), str(step.text), exit, self.indent(stdout), self.indent(stderr))) self.remember_step_timing( '%s %s' % (step.what, step.text), stopped - started) @@ -321,16 +337,10 @@ class YarnRunner(cliapp.Application): self.snapshot_datadir(self.tempdir, step_env['DATADIR'], scenario, step_number, step) - def scenario_dir(self, tempdir, scenario): - return os.path.join(tempdir, self.nice(scenario.name)) - - def datadir(self, tempdir, scenario): - sd = self.scenario_dir(tempdir, scenario) - return os.path.join(sd, 'datadir') - def snapshot_dir(self, tempdir, scenario, step, step_number): - sd = self.scenario_dir(tempdir, scenario) - base = '%03d-%s-%s' % (step_number, step.what, self.nice(step.text)) + sd = yarnlib.ScenarioRunner.scenario_dir(tempdir, scenario) + base = '%03d-%s-%s' % (step_number, step.what, + yarnlib.ScenarioRunner.nice(step.text)) return os.path.join(sd, base) def snapshot_datadir(self, tempdir, datadir, scenario, step_number, step): @@ -340,21 +350,6 @@ class YarnRunner(cliapp.Application): if exit != 0: logging.warning('Snapshot copy failed:\n%s\n%s' % (out, err)) - def nice(self, name): - # Quote a scenario or step name so it forms a nice filename. - nice_chars = "abcdefghijklmnopqrstuvwxyz" - nice_chars += nice_chars.upper() - nice_chars += "0123456789-." - - nice = [] - for c in name: - if c in nice_chars: - nice.append(c) - elif not nice or nice[-1] != '_': - nice.append('_') - nice = ''.join(nice) - return nice - def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) |