diff options
Diffstat (limited to 'yarn')
| -rwxr-xr-x | yarn | 277 |
1 files changed, 87 insertions, 190 deletions
@@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2013 Lars Wirzenius +# Copyright 2013-2014 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -79,6 +79,11 @@ class YarnRunner(cliapp.Application): ['timings'], 'report wall clock time for each scenario and step') + self.settings.boolean( + ['allow-missing-steps'], + 'allow scenarios to reference steps that do not exist, ' + 'by warning about them, but otherwise ignoring the scenarios') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -109,32 +114,50 @@ class YarnRunner(cliapp.Application): self.ts = ttystatus.TerminalStatus(period=0.001) if not self.settings['quiet'] and not self.settings['verbose']: self.ts.format( - '%ElapsedTime() %Index(scenario,scenarios): ' + '%ElapsedTime() %Index(current_step,all_steps): ' '%String(scenario_name): ' - 'step %Index(step,steps): %String(step_name)') + '%String(step_name)') + + if self.settings['tempdir']: + self.tempdir = os.path.abspath(self.settings['tempdir']) + if not os.path.exists(self.tempdir): + os.mkdir(self.tempdir) + else: + self.tempdir = tempfile.mkdtemp() scenarios, implementations = self.parse_scenarios(args) - self.check_there_are_scenarios(scenarios) - self.check_for_duplicate_scenario_names(scenarios) - self.check_for_thens(scenarios) - self.connect_implementations(scenarios, implementations) + sv = yarnlib.ScenarioValidator(scenarios) + sv.validate_all() + scenarios = self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() - self.ts['scenarios'] = scenarios - self.ts['num_scenarios'] = len(scenarios) self.info('Found %d scenarios' % len(scenarios)) + all_steps = [] + for scenario in scenarios: + all_steps.extend(scenario.steps) + self.ts['all_steps'] = all_steps + self.scenarios_run = 0 + self.skipped_for_assuming = 0 self.steps_run = 0 self.timings = [] + scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(), + self.parse_env(), + pre_step_cb=self.pre_step, + post_step_cb=self.post_step) + start_time = time.time() failed_scenarios = [] for scenario in self.select_scenarios(scenarios): - if not self.run_scenario(scenario, shell_prelude): + if not self.run_scenario(scenario_runner, scenario): failed_scenarios.append(scenario) duration = time.time() - start_time + if not self.settings['snapshot']: + shutil.rmtree(self.tempdir) + if not self.settings['quiet']: self.ts.clear() self.ts.finish() @@ -149,6 +172,9 @@ class YarnRunner(cliapp.Application): '(%d total steps), ' 'in %.1f seconds' % (self.scenarios_run, self.steps_run, duration)) + if self.skipped_for_assuming: + print ('Scenarios SKIPPED due to ASSUMING step failing: %d' + % self.skipped_for_assuming) if self.settings['timings']: self.report_timings() @@ -166,76 +192,28 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations - def check_there_are_scenarios(self, scenarios): - if not scenarios: - raise cliapp.AppException( - 'There are no scenarios; must have at least one.') - - def check_for_duplicate_scenario_names(self, scenarios): - counts = collections.Counter() - for s in scenarios: - counts[s.name] += 1 - - duplicates = [name for name in counts if counts[name] > 1] - if duplicates: - duplist = ''.join(' %s\n' % name for name in duplicates) - raise cliapp.AppException( - 'There are scenarios with duplicate names:\n%s' % duplist) - - def check_for_thens(self, scenarios): - no_thens = [] - for scenario in scenarios: - for step in scenario.steps: - if step.what == 'THEN': - break - else: - no_thens.append(scenario) - - if no_thens: - raise cliapp.AppException( - 'Some scenarios have no THENs:\n%s' % - ''.join(' "%s"\n' % s.name for s in scenarios)) - def connect_implementations(self, scenarios, implementations): - for scenario in scenarios: - for step in scenario.steps: - self.connect_implementation(scenario, step, implementations) - - def connect_implementation(self, scenario, step, implementations): - matching = [i for i in implementations - if step.what == i.what and - self.implements_matches_step(i, step)] - - if len(matching) == 0: - raise cliapp.AppException( - 'Scenario "%s", step "%s %s" has no matching ' - 'implementation' % - (scenario.name, step.what, step.text)) - if len(matching) > 1: - s = '\n'.join( - 'IMPLEMENTS %s %s' % (i.what, i.regexp) - for i in matching) - raise cliapp.AppException( - 'Scenario "%s", step "%s %s" has more than one ' - 'matching implementations:\n%s' % - (scenario.name, step.what, step.text, s)) + if self.settings['allow-missing-steps']: + def warn_missing(scenario, step): + self.warning( + 'Scenario %s has missing step %s %s' % + (scenario.name, step.what, step.text)) + return True + step_connector = yarnlib.ScenarioStepConnector( + implementations, missing_step_cb=warn_missing) + else: + step_connector = yarnlib.ScenarioStepConnector(implementations) - assert step.implementation is None - step.implementation = matching[0] + return step_connector.connect_implementations(scenarios) def load_shell_libraries(self): if not self.settings['shell-library']: self.info('No shell libraries defined') return '' - libs = [] - for filename in self.settings['shell-library']: - self.info('Loading shell library %s' % filename) - with open(filename) as f: - text = f.read() - libs.append('# Loaded from %s\n\n%s\n\n' % (filename, text)) - - return ''.join(libs) + return yarnlib.load_shell_libraries( + self.settings['shell-library'], + pre_read_cb=lambda fn: self.info('Loading shell library %s' % fn)) def select_scenarios(self, scenarios): @@ -256,133 +234,60 @@ class YarnRunner(cliapp.Application): return scenarios - def run_scenario(self, scenario, shell_prelude): + def run_scenario(self, scenario_runner, scenario): self.start_scenario_timing(scenario.name) started = time.time() self.info('Running scenario %s' % scenario.name) - self.ts['scenario'] = scenario self.ts['scenario_name'] = scenario.name - self.ts['steps'] = scenario.steps self.scenarios_run += 1 if self.settings['no-act']: self.info('Pretending everything went OK') + for step in scenario.steps: + self.ts['current_step'] = step self.remember_scenario_timing(time.time() - started) return True - if self.settings['tempdir']: - tempdir = os.path.abspath(self.settings['tempdir']) - if not os.path.exists(tempdir): - os.mkdir(tempdir) - else: - tempdir = tempfile.mkdtemp() - - os.mkdir(self.scenario_dir(tempdir, scenario)) - datadir = self.datadir(tempdir, scenario) + os.mkdir(self.scenario_dir(self.tempdir, scenario)) + datadir = self.datadir(self.tempdir, scenario) os.mkdir(datadir) self.info('DATADIR is %s' % datadir) + homedir = self.homedir(datadir) + os.mkdir(homedir) + self.info('HOME for tests is %s' % homedir) - assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] - cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] - normal = [s for s in scenario.steps if s not in assuming + cleanup] - - ok = True - step_number = 0 - - for step in assuming: - exit = self.run_step(datadir, scenario, step, shell_prelude, False) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - self.ts.notify( - 'Skipping "%s" because "%s %s" failed' % - (scenario.name, step.what, step.text)) - break - else: - for step in normal: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - for step in cleanup: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - if not self.settings['snapshot']: - shutil.rmtree(tempdir) + ok = scenario_runner.run_scenario(scenario, datadir, homedir) self.remember_scenario_timing(time.time() - started) return ok - def clean_env(self): - '''Return a clean environment for running tests.''' - - whitelisted = [ - 'PATH', - ] - - hardcoded = { - 'TERM': 'dumb', - 'SHELL': '/bin/sh', - 'LC_ALL': 'C', - 'USER': 'tomjon', - 'USERNAME': 'tomjon', - 'LOGNAME': 'tomjon', - 'HOME': '/this/path/does/not/exist', - } - - env = {} - - for key in whitelisted: - if key in os.environ: - env[key] = os.environ[key] - - for key in hardcoded: - env[key] = hardcoded[key] + def homedir(self, datadir): + return os.path.join(datadir, 'HOME') + def parse_env(self): for option_arg in self.settings['env']: if '=' not in option_arg: raise cliapp.AppException( '--env argument must contain "=" ' 'to separate environment variable name and value') key, value = option_arg.split('=', 1) - env[key] = value + yield key, value - return env - - def run_step(self, datadir, scenario, step, shell_prelude, report_error): + def pre_step(self, step, **ignored): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) - self.ts['step'] = step + self.ts['current_step'] = step self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 - m = self.implements_matches_step(step.implementation, step) - assert m is not None - env = self.clean_env() - env['DATADIR'] = datadir - env['SRCDIR'] = os.getcwd() - for i, match in enumerate(m.groups('')): - env['MATCH_%d' % (i+1)] = match + return (started,) - shell_script = '%s\n\n%s\n' % ( - shell_prelude, step.implementation.shell) - exit, stdout, stderr = cliapp.runcmd_unchecked( - ['sh', '-xeuc', shell_script], env=env) + def post_step(self, scenario, step, step_number, step_env, + exit, stdout, stderr, pre_step_userdata): + stopped = time.time() + (started,) = pre_step_userdata logging.debug('Exit code: %d' % exit) if stdout: @@ -393,20 +298,24 @@ class YarnRunner(cliapp.Application): logging.debug('Standard error:\n%s' % self.indent(stderr)) else: logging.debug('Standard error: empty') - - if exit != 0 and report_error: - self.error( - 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' - 'with exit code %d:\n' - 'Standard output from shell command:\n%s' - 'Standard error from shell command:\n%s' % - (scenario.name, step.what, step.text, exit, - self.indent(stdout), self.indent(stderr))) - + if exit != 0: + if step.what == 'ASSUMING': + self.ts.notify( + 'Skipping "%s" because "%s %s" failed' % + (scenario.name, step.what, step.text)) + self.skipped_for_assuming += 1 + else: + self.error( + 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' + 'with exit code %d:\n' + 'Standard output from shell command:\n%s' + 'Standard error from shell command:\n%s' % + (scenario.name, step.what, step.text, exit, + self.indent(stdout), self.indent(stderr))) self.remember_step_timing( - '%s %s' % (step.what, step.text), time.time() - started) - - return exit + '%s %s' % (step.what, step.text), stopped - started) + self.snapshot_datadir(self.tempdir, step_env['DATADIR'], + scenario, step_number, step) def scenario_dir(self, tempdir, scenario): return os.path.join(tempdir, self.nice(scenario.name)) @@ -439,18 +348,6 @@ class YarnRunner(cliapp.Application): nice = ''.join(nice) return nice - def implements_matches_step(self, implements, step): - '''Return re.Match if implements matches the step. - - Otherwise, return None. - - ''' - - m = re.match(implements.regexp, step.text, re.I) - if m and m.end() != len(step.text): - return None - return m - def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) |