#!/usr/bin/env python3 # # Copyright (c) 2013 German M. Bravo (Kronuz) # Copyright (c) 2018 Codethink Limited # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # This file is substantially based on German's work, obtained at: # https://github.com/Kronuz/ansi2html.git # import os import sys import re import shlex import subprocess from collections import Mapping from contextlib import contextmanager from tempfile import TemporaryDirectory import click from buildstream import _yaml from buildstream import utils from buildstream._exceptions import BstError _ANSI2HTML_STYLES = {} ANSI2HTML_CODES_RE = re.compile('(?:\033\\[(\d+(?:;\d+)*)?([cnRhlABCDfsurgKJipm]))') ANSI2HTML_PALETTE = { # See http://ethanschoonover.com/solarized 'solarized': ['#073642', '#D30102', '#859900', '#B58900', '#268BD2', '#D33682', '#2AA198', '#EEE8D5', '#002B36', '#CB4B16', '#586E75', '#657B83', '#839496', '#6C71C4', '#93A1A1', '#FDF6E3'], # Above mapped onto the xterm 256 color palette 'solarized-xterm': ['#262626', '#AF0000', '#5F8700', '#AF8700', '#0087FF', '#AF005F', '#00AFAF', '#E4E4E4', '#1C1C1C', '#D75F00', '#585858', '#626262', '#808080', '#5F5FAF', '#8A8A8A', '#FFFFD7'], # Gnome default: 'tango': ['#000000', '#CC0000', '#4E9A06', '#C4A000', '#3465A4', '#75507B', '#06989A', '#D3D7CF', '#555753', '#EF2929', '#8AE234', '#FCE94F', '#729FCF', '#AD7FA8', '#34E2E2', '#EEEEEC'], # xterm: 'xterm': ['#000000', '#CD0000', '#00CD00', '#CDCD00', '#0000EE', '#CD00CD', '#00CDCD', '#E5E5E5', '#7F7F7F', '#FF0000', '#00FF00', '#FFFF00', '#5C5CFF', '#FF00FF', '#00FFFF', '#FFFFFF'], 'console': ['#000000', '#AA0000', '#00AA00', '#AA5500', '#0000AA', '#AA00AA', '#00AAAA', '#AAAAAA', '#555555', '#FF5555', '#55FF55', '#FFFF55', '#5555FF', '#FF55FF', '#55FFFF', '#FFFFFF'], } def _ansi2html_get_styles(palette): if palette not in _ANSI2HTML_STYLES: p = ANSI2HTML_PALETTE.get(palette, ANSI2HTML_PALETTE['console']) regular_style = { '1': '', # bold '2': 'opacity:0.5', '4': 'text-decoration:underline', '5': 'font-weight:bold', '7': '', '8': 'display:none', } bold_style = regular_style.copy() for i in range(8): regular_style['3%s' % i] = 'color:%s' % p[i] regular_style['4%s' % i] = 'background-color:%s' % p[i] bold_style['3%s' % i] = 'color:%s' % p[i + 8] bold_style['4%s' % i] = 'background-color:%s' % p[i + 8] # The default xterm 256 colour p: indexed_style = {} for i in range(16): indexed_style['%s' % i] = p[i] for rr in range(6): for gg in range(6): for bb in range(6): i = 16 + rr * 36 + gg * 6 + bb r = (rr * 40 + 55) if rr else 0 g = (gg * 40 + 55) if gg else 0 b = (bb * 40 + 55) if bb else 0 indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (r, g, b)) for g in range(24): i = g + 232 l = g * 10 + 8 indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (l, l, l)) _ANSI2HTML_STYLES[palette] = (regular_style, bold_style, indexed_style) return _ANSI2HTML_STYLES[palette] def ansi2html(text, palette='solarized'): def _ansi2html(m): if m.group(2) != 'm': return '' import sys state = None sub = '' cs = m.group(1) cs = cs.strip() if cs else '' for c in cs.split(';'): c = c.strip().lstrip('0') or '0' if c == '0': while stack: sub += '' stack.pop() elif c in ('38', '48'): extra = [c] state = 'extra' elif state == 'extra': if c == '5': state = 'idx' elif c == '2': state = 'r' elif state: if state == 'idx': extra.append(c) state = None # 256 colors color = indexed_style.get(c) # TODO: convert index to RGB! if color is not None: sub += '' % ('color' if extra[0] == '38' else 'background-color', color) stack.append(extra) elif state in ('r', 'g', 'b'): extra.append(c) if state == 'r': state = 'g' elif state == 'g': state = 'b' else: state = None try: color = '#' + ''.join( '%02X' % c if 0 <= c <= 255 else None for x in extra for c in [int(x)] ) except (ValueError, TypeError): pass else: sub += ''.format( 'color' if extra[0] == '38' else 'background-color', color) stack.append(extra) else: if '1' in stack: style = bold_style.get(c) else: style = regular_style.get(c) if style is not None: sub += '' % style # Still needs to be added to the stack even if style is empty # (so it can check '1' in stack above, for example) stack.append(c) return sub stack = [] regular_style, bold_style, indexed_style = _ansi2html_get_styles(palette) sub = ANSI2HTML_CODES_RE.sub(_ansi2html, text) while stack: sub += '' stack.pop() return sub # workdir() # # Sets up a new temp directory with a config file # # Args: # work_directory (str): The directory where to create a tempdir first # source_cache (str): The directory of a source cache to share with, or None # # Yields: # The buildstream.conf full path # @contextmanager def workdir(source_cache=None): with TemporaryDirectory(prefix='run-bst-', dir=os.getcwd()) as tempdir: if not source_cache: source_cache = os.path.join(tempdir, 'sources') bst_config_file = os.path.join(tempdir, 'buildstream.conf') config = { 'sourcedir': source_cache, 'artifactdir': os.path.join(tempdir, 'artifacts'), 'logdir': os.path.join(tempdir, 'logs'), 'builddir': os.path.join(tempdir, 'build'), } _yaml.dump(config, bst_config_file) yield (tempdir, bst_config_file, source_cache) # run_command() # # Runs a command # # Args: # config_file (str): The path to the config file to use # directory (str): The project directory # command (str): A command string # # Returns: # (str): The colorized combined stdout/stderr of BuildStream # def run_command(config_file, directory, command): click.echo("Running command in directory '{}': bst {}".format(directory, command), err=True) argv = ['bst', '--colors', '--config', config_file] + shlex.split(command) p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = p.communicate() return out.decode('utf-8').strip() # generate_html # # Generate html based on the output # # Args: # output (str): The output of the BuildStream command # directory (str): The project directory # config_file (str): The config file # source_cache (str): The source cache # tempdir (str): The base work directory # palette (str): The rendering color style # command (str): The command # fake_output (bool): Whether the provided output is faked or not # # Returns: # (str): The html formatted output # def generate_html(output, directory, config_file, source_cache, tempdir, palette, command, fake_output): test_base_name = os.path.basename(directory) if fake_output: show_command = command else: show_command = 'bst ' + command # Substitute some things we want normalized for the docs output = re.sub(os.environ.get('HOME'), '/home/user', output) output = re.sub(config_file, '/home/user/.config/buildstream.conf', output) output = re.sub(source_cache, '/home/user/.cache/buildstream/sources', output) output = re.sub(tempdir, '/home/user/.cache/buildstream', output) output = re.sub(directory, '/home/user/{}'.format(test_base_name), output) # Now convert to HTML and add some surrounding sugar output = ansi2html(output, palette=palette) # Finally format it nicely into a
final_output = '\n' + \ '
' + \ '
\n' + \
                   'user@host:' + \
                   '~/{}$ '.format(test_base_name) + \
                   show_command + '\n'

    if output:
        final_output += '\n' + output + '\n'

    final_output += '
\n' return final_output # check_needs_build() # # Checks whether filename, specified relative to basedir, # needs to be built (based on whether it exists). # # Args: # basedir (str): The base directory to check relative of, or None for CWD # filename (str): The basedir relative path to the file # force (bool): Whether force rebuilding of existing things is enabled # # Returns: # (bool): Whether the file needs to be built # def check_needs_build(basedir, filename, force=False): if force: return True if basedir is None: basedir = os.getcwd() filename = os.path.join(basedir, filename) filename = os.path.realpath(filename) if not os.path.exists(filename): return True return False def run_session(description, tempdir, source_cache, palette, config_file, force): desc = _yaml.load(description, shortname=os.path.basename(description)) desc_dir = os.path.dirname(description) # Preflight commands and check if we can skip this session # if not force: needs_build = False commands = _yaml.node_get(desc, list, 'commands') for command in commands: output = _yaml.node_get(command, str, 'output', default_value=None) if output is not None and check_needs_build(desc_dir, output, force=False): needs_build = True break if not needs_build: click.echo("Skipping '{}' as no files need to be built".format(description), err=True) return # FIXME: Workaround a setuptools bug where the symlinks # we store in git dont get carried into a release # tarball. This workaround lets us build docs from # a source distribution tarball. # symlinks = _yaml.node_get(desc, Mapping, 'workaround-symlinks', default_value={}) for symlink, target in _yaml.node_items(symlinks): # Resolve real path to where symlink should be symlink = os.path.join(desc_dir, symlink) # Ensure dir exists symlink_dir = os.path.dirname(symlink) os.makedirs(symlink_dir, exist_ok=True) click.echo("Generating symlink at: {} (target: {})".format(symlink, target), err=True) # Generate a symlink try: os.symlink(target, symlink) except FileExistsError: # If the files exist, we're running from a git checkout and # not a source distribution, no need to complain pass remove_files = _yaml.node_get(desc, list, 'remove-files', default_value=[]) for remove_file in remove_files: remove_file = os.path.join(desc_dir, remove_file) remove_file = os.path.realpath(remove_file) if os.path.isdir(remove_file): utils._force_rmtree(remove_file) else: utils.safe_remove(remove_file) # Run commands # commands = _yaml.node_get(desc, list, 'commands') for c in commands: command = _yaml.node_get(desc, Mapping, 'commands', indices=[commands.index(c)]) # Get the directory where this command should be run directory = _yaml.node_get(command, str, 'directory') directory = os.path.join(desc_dir, directory) directory = os.path.realpath(directory) # Get the command string command_str = _yaml.node_get(command, str, 'command') # Check if there is fake output command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None) # Run the command, or just use the fake output if command_fake_output is None: command_out = run_command(config_file, directory, command_str) else: command_out = command_fake_output # Encode and save the output if that was asked for output = _yaml.node_get(command, str, 'output', default_value=None) if output is not None: # Convert / Generate a nice
converted = generate_html(command_out, directory, config_file, source_cache, tempdir, palette, command_str, command_fake_output is not None) # Save it filename = os.path.join(desc_dir, output) filename = os.path.realpath(filename) output_dir = os.path.dirname(filename) os.makedirs(output_dir, exist_ok=True) with open(filename, 'wb') as f: f.write(converted.encode('utf-8')) click.echo("Saved session at '{}'".format(filename), err=True) @click.command(short_help="Run a bst command and capture stdout/stderr in html") @click.option('--directory', '-C', type=click.Path(file_okay=False, dir_okay=True), help="The project directory where to run the command") @click.option('--force', is_flag=True, default=False, help="Force rebuild, even if the file exists") @click.option('--source-cache', type=click.Path(file_okay=False, dir_okay=True), help="A shared source cache") @click.option('--palette', '-p', default='tango', type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']), help="Selects a palette for the output style") @click.option('--output', '-o', type=click.Path(file_okay=True, dir_okay=False, writable=True), help="A file to store the output") @click.option('--description', '-d', type=click.Path(file_okay=True, dir_okay=False, readable=True), help="A file describing what to do") @click.argument('command', type=click.STRING, nargs=-1) def run_bst(directory, force, source_cache, description, palette, output, command): """Run a bst command and capture stdout/stderr in html This command normally takes a description yaml file, see the HACKING file for information on it's format. """ if not source_cache and os.environ.get('BST_SOURCE_CACHE'): source_cache = os.environ['BST_SOURCE_CACHE'] if output is not None and not check_needs_build(None, output, force=force): click.echo("No need to rebuild {}".format(output)) return 0 with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache): if description: run_session(description, tempdir, source_cache, palette, config_file, force) return 0 # Run a command specified on the CLI # if not directory: directory = os.getcwd() else: directory = os.path.abspath(directory) directory = os.path.realpath(directory) if not command: command = [] command_str = " ".join(command) # Run the command # command_out = run_command(config_file, directory, command_str) # Generate a nice html div for this output # converted = generate_html(command_out, directory, config_file, source_cache, tempdir, palette, command_str) if output is None: click.echo(converted) else: outdir = os.path.dirname(output) os.makedirs(outdir, exist_ok=True) with open(output, 'wb') as f: f.write(converted.encode('utf-8')) return 0 if __name__ == '__main__': try: run_bst() except BstError as e: click.echo("Error: {}".format(e), err=True) sys.exit(-1)