#!/usr/bin/env python3
#
# Copyright (C) 2017 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see .
#
# Authors:
# Tristan Van Berkom
import datetime
import os
from collections import OrderedDict
from contextlib import ExitStack
from mmap import mmap
import click
import pkg_resources
from ruamel import yaml
from . import Profile
from .. import Element, Scope, Consistency
from .. import _yaml
from .._exceptions import ImplError
from .._message import MessageType
from ..plugin import _plugin_lookup
# Widget()
#
# Args:
# content_profile (Profile): The profile to use for rendering content
# format_profile (Profile): The profile to use for rendering formatting
#
# An abstract class for printing output columns in our text UI.
#
class Widget():
def __init__(self, content_profile, format_profile):
# The content profile
self.content_profile = content_profile
# The formatting profile
self.format_profile = format_profile
# size_request()
#
# Gives the widget a chance to preflight the pipeline
# and figure out what size it might need for alignment purposes
#
# Args:
# pipeline (Pipeline): The pipeline to process
#
def size_request(self, pipeline):
pass
# render()
#
# Renders a string to be printed in the UI
#
# Args:
# message (Message): A message to print
#
# Returns:
# (str): The string this widget prints for the given message
#
def render(self, message):
raise ImplError("{} does not implement render()".format(type(self).__name__))
# Used to add spacing between columns
class Space(Widget):
def render(self, message):
return ' '
# A widget for rendering the debugging column
class Debug(Widget):
def render(self, message):
unique_id = 0 if message.unique_id is None else message.unique_id
text = self.format_profile.fmt('[pid:')
text += self.content_profile.fmt("{: <5}".format(message.pid))
text += self.format_profile.fmt(" id:")
text += self.content_profile.fmt("{:0>3}".format(unique_id))
text += self.format_profile.fmt(']')
return text
# A widget for rendering the time codes
class TimeCode(Widget):
def __init__(self, content_profile, format_profile, brackets=True):
self.brackets = brackets
super(TimeCode, self).__init__(content_profile, format_profile)
def render(self, message):
return self.render_time(message.elapsed)
def render_time(self, elapsed):
if elapsed is None:
fields = [
self.content_profile.fmt('--')
for i in range(3)
]
else:
hours, remainder = divmod(int(elapsed.total_seconds()), 60 * 60)
minutes, seconds = divmod(remainder, 60)
fields = [
self.content_profile.fmt("{0:02d}".format(field))
for field in [hours, minutes, seconds]
]
text = ''
if self.brackets:
text += self.format_profile.fmt('[')
text += self.format_profile.fmt(':').join(fields)
if self.brackets:
text += self.format_profile.fmt(']')
return text
# A widget for rendering the MessageType
class TypeName(Widget):
action_colors = {
MessageType.DEBUG: "cyan",
MessageType.STATUS: "cyan",
MessageType.INFO: "magenta",
MessageType.WARN: "yellow",
MessageType.ERROR: "red",
MessageType.START: "blue",
MessageType.SUCCESS: "green",
MessageType.FAIL: "red",
MessageType.BUG: "red",
}
def render(self, message):
return self.content_profile.fmt("{: <7}"
.format(message.message_type.upper()),
bold=True, dim=True,
fg=self.action_colors[message.message_type])
# A widget for displaying the Element name
class ElementName(Widget):
def __init__(self, content_profile, format_profile):
super(ElementName, self).__init__(content_profile, format_profile)
# Pre initialization format string, before we know the length of
# element names in the pipeline
self.fmt_string = '{: <30}'
def size_request(self, pipeline):
longest_name = 0
for plugin in pipeline.dependencies(Scope.ALL, include_sources=True):
longest_name = max(len(plugin.name), longest_name)
# Put a cap at a specific width, usually some elements cause the line
# to be too long, just live with the unaligned columns in that case
longest_name = min(longest_name, 30)
self.fmt_string = '{: <' + str(longest_name) + '}'
def render(self, message):
element_id = message.task_id or message.unique_id
if element_id is None:
return ""
plugin = _plugin_lookup(element_id)
name = plugin.name
# Sneak the action name in with the element name
action_name = message.action_name
if not action_name:
action_name = "Main"
return self.format_profile.fmt('[') + \
self.content_profile.fmt("{: >5}".format(action_name.lower())) + \
self.format_profile.fmt(':') + \
self.content_profile.fmt(self.fmt_string.format(name)) + \
self.format_profile.fmt(']')
# A widget for displaying the primary message text
class MessageText(Widget):
def render(self, message):
return message.message
# A widget for formatting the element cache key
class CacheKey(Widget):
def __init__(self, content_profile, format_profile, err_profile):
super(CacheKey, self).__init__(content_profile, format_profile)
self.err_profile = err_profile
self.key_length = 0
def size_request(self, pipeline):
self.key_length = pipeline.context.log_key_length
def render(self, message):
# This can only happen when logging before initialization in debug mode
if not self.key_length:
return self.format_profile.fmt('[') + (' ' * 8) + self.format_profile.fmt(']')
missing = False
key = ' ' * self.key_length
element_id = message.task_id or message.unique_id
if element_id is None:
return ""
plugin = _plugin_lookup(element_id)
if isinstance(plugin, Element):
_, key, missing = plugin._get_full_display_key()
if message.message_type in [MessageType.FAIL, MessageType.BUG]:
text = self.err_profile.fmt(key)
else:
text = self.content_profile.fmt(key, dim=missing)
return self.format_profile.fmt('[') + text + self.format_profile.fmt(']')
# A widget for formatting the log file
class LogFile(Widget):
def __init__(self, content_profile, format_profile, err_profile):
super(LogFile, self).__init__(content_profile, format_profile)
self.err_profile = err_profile
self.logdir = ''
def size_request(self, pipeline):
# Hold on to the logging directory so we can abbreviate
self.logdir = pipeline.context.logdir
def render(self, message):
if message.logfile and message.scheduler:
logfile = message.logfile
if logfile.startswith(self.logdir):
logfile = logfile[len(self.logdir) + 1:]
if message.message_type in [MessageType.FAIL, MessageType.BUG]:
text = self.err_profile.fmt(logfile)
else:
text = self.content_profile.fmt(logfile, dim=True)
else:
text = ''
return text
# A widget for formatting a log line
class LogLine(Widget):
def __init__(self, content_profile, format_profile, success_profile, err_profile, detail_profile,
indent=4,
log_lines=10,
message_lines=10,
debug=False):
super(LogLine, self).__init__(content_profile, format_profile)
self.columns = []
self.success_profile = success_profile
self.err_profile = err_profile
self.detail_profile = detail_profile
self.indent = ' ' * indent
self.log_lines = log_lines
self.message_lines = message_lines
self.space_widget = Space(content_profile, format_profile)
self.message_widget = MessageText(content_profile, format_profile)
self.logfile_widget = LogFile(content_profile, format_profile, err_profile)
if debug:
self.columns.extend([
Debug(content_profile, format_profile)
])
self.columns.extend([
TimeCode(content_profile, format_profile),
CacheKey(content_profile, format_profile, err_profile),
ElementName(content_profile, format_profile),
self.space_widget,
TypeName(content_profile, format_profile),
self.space_widget
])
def size_request(self, pipeline):
for widget in self.columns:
widget.size_request(pipeline)
self.space_widget.size_request(pipeline)
self.message_widget.size_request(pipeline)
self.logfile_widget.size_request(pipeline)
def render(self, message):
# Render the column widgets first
text = ''
for widget in self.columns:
text += widget.render(message)
# Show the log file only in the main start/success/fail messages
if message.logfile and message.scheduler:
text += self.logfile_widget.render(message)
else:
text += self.message_widget.render(message)
text += '\n'
extra_nl = False
# Now add some custom things
if message.detail is not None:
# Split and truncate message detail down to message_lines lines
lines = message.detail.splitlines(True)
n_lines = len(lines)
abbrev = False
if message.message_type not in [MessageType.FAIL, MessageType.BUG] \
and n_lines > self.message_lines:
abbrev = True
lines = lines[0:self.message_lines]
else:
lines[n_lines - 1] = lines[n_lines - 1].rstrip('\n')
detail = self.indent + self.indent.join(lines)
text += '\n'
if message.message_type in [MessageType.FAIL, MessageType.BUG]:
text += self.err_profile.fmt(detail, bold=True)
else:
text += self.detail_profile.fmt(detail)
if abbrev:
text += self.indent + \
self.content_profile.fmt('Message contains {} additional lines'
.format(n_lines - self.message_lines), dim=True)
text += '\n'
extra_nl = True
if message.sandbox is not None:
sandbox = self.indent + 'Sandbox directory: ' + message.sandbox
text += '\n'
if message.message_type == MessageType.FAIL:
text += self.err_profile.fmt(sandbox, bold=True)
else:
text += self.detail_profile.fmt(sandbox)
text += '\n'
extra_nl = True
if message.scheduler and message.message_type == MessageType.FAIL:
log_content = self.read_last_lines(message.logfile)
log_content = self.indent + self.indent.join(log_content.splitlines(True))
text += '\n'
text += self.detail_profile.fmt(log_content)
text += '\n'
extra_nl = True
if extra_nl:
text += '\n'
return text
def read_last_lines(self, logfile):
with ExitStack() as stack:
# mmap handles low-level memory details, allowing for
# faster searches
f = stack.enter_context(open(logfile, 'r+'))
log = stack.enter_context(mmap(f.fileno(), os.path.getsize(f.name)))
count = 0
end = log.size() - 1
while count < self.log_lines and end >= 0:
location = log.rfind(b'\n', 0, end)
count += 1
# If location is -1 (none found), this will print the
# first character despite the later +1
end = location
# end+1 since we do not want to print the first newline
# (consistent with `tail` behavior)
lines = log[end:].splitlines()
return '\n'.join([line.decode('utf-8') for line in lines]).rstrip()
#
# A message to be printed at program startup, indicating
# some things about user configuration and BuildStream version
# and so on.
#
# Args:
# pipeline (Pipeline): The pipeline to print the heading of
# log_file (file): An optional file handle for additional logging
# deps (list): Optional list of elements, default is to use the whole pipeline
# styling (bool): Whether to enable ansi escape codes in the output
#
def print_heading(self, pipeline, log_file, deps=None, styling=False):
context = pipeline.context
project = pipeline.project
starttime = datetime.datetime.now()
bst = pkg_resources.require("buildstream")[0]
text = ''
# Main invocation context
text += '\n'
text += self.content_profile.fmt("BuildStream Version {}\n".format(bst.version), bold=True)
values = OrderedDict()
values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
values["Project"] = "{} ({})".format(project.name, project.directory)
values["Targets"] = ", ".join([t.name for t in pipeline.targets])
text += self.format_values(values)
# User configurations
text += '\n'
text += self.content_profile.fmt("User Configuration\n", bold=True)
values = OrderedDict()
values["Configuration File"] = \
"Default Configuration" if not context.config_origin else context.config_origin
values["Log Files"] = context.logdir
values["Source Mirrors"] = context.sourcedir
values["Build Area"] = context.builddir
values["Artifact Cache"] = context.artifactdir
values["Maximum Fetch Tasks"] = context.sched_fetchers
values["Maximum Build Tasks"] = context.sched_builders
values["Maximum Push Tasks"] = context.sched_pushers
values["Maximum Network Retries"] = context.sched_network_retries
text += self.format_values(values)
text += '\n'
# Project Options
if len(project._options.variables) > 0:
text += self.content_profile.fmt("Project Options\n", bold=True)
values = OrderedDict()
for key in sorted(project._options.variables):
values[key] = project._options.variables[key]
text += self.format_values(values)
text += '\n'
# Plugins
text += self.format_plugins(pipeline.element_factory.loaded_dependencies,
pipeline.source_factory.loaded_dependencies)
# Pipeline state
text += self.content_profile.fmt("Pipeline\n", bold=True)
if deps is None:
deps = pipeline.dependencies(Scope.ALL)
text += self.show_pipeline(deps, context.log_element_format)
text += '\n'
# Separator line before following output
text += self.format_profile.fmt("~" * 79 + '\n')
click.echo(text, color=styling, nl=False)
if log_file:
click.echo(text, file=log_file, color=False, nl=False)
# Print queue summaries at the end of a scheduler run
#
def print_summary(self, pipeline, scheduler, log_file, styling=False):
# Early silent return if there are no queues, can happen
# only in the case that the pipeline early returned due to
# an inconsistent pipeline state.
if scheduler.queues is None:
return
text = self.content_profile.fmt("Pipeline Summary\n", bold=True)
values = OrderedDict()
values['Total'] = self.content_profile.fmt(str(pipeline.total_elements))
values['Session'] = self.content_profile.fmt(str(pipeline.session_elements))
processed_maxlen = 1
skipped_maxlen = 1
failed_maxlen = 1
for queue in scheduler.queues:
processed_maxlen = max(len(str(len(queue.processed_elements))), processed_maxlen)
skipped_maxlen = max(len(str(len(queue.skipped_elements))), skipped_maxlen)
failed_maxlen = max(len(str(len(queue.failed_elements))), failed_maxlen)
for queue in scheduler.queues:
processed = str(len(queue.processed_elements))
skipped = str(len(queue.skipped_elements))
failed = str(len(queue.failed_elements))
processed_align = ' ' * (processed_maxlen - len(processed))
skipped_align = ' ' * (skipped_maxlen - len(skipped))
failed_align = ' ' * (failed_maxlen - len(failed))
status_text = self.content_profile.fmt("processed ") + \
self.success_profile.fmt(processed) + \
self.format_profile.fmt(', ') + processed_align
status_text += self.content_profile.fmt("skipped ") + \
self.content_profile.fmt(skipped) + \
self.format_profile.fmt(', ') + skipped_align
status_text += self.content_profile.fmt("failed ") + \
self.err_profile.fmt(failed) + ' ' + failed_align
values["{} Queue".format(queue.action_name)] = status_text
text += self.format_values(values, style_value=False)
click.echo(text, color=styling, nl=False)
if log_file:
click.echo(text, file=log_file, color=False, nl=False)
def format_plugins(self, element_plugins, source_plugins):
text = ""
if not (element_plugins or source_plugins):
return text
text += self.content_profile.fmt("Loaded Plugins\n", bold=True)
if element_plugins:
text += self.format_profile.fmt(" Element Plugins\n")
for plugin in element_plugins:
text += self.content_profile.fmt(" - {}\n".format(plugin))
if source_plugins:
text += self.format_profile.fmt(" Source Plugins\n")
for plugin in source_plugins:
text += self.content_profile.fmt(" - {}\n".format(plugin))
text += '\n'
return text
def format_values(self, values, style_value=True):
text = ''
max_key_len = 0
for key, value in values.items():
max_key_len = max(len(key), max_key_len)
for key, value in values.items():
text += self.format_profile.fmt(" {}: {}".format(key, ' ' * (max_key_len - len(key))))
if style_value:
text += self.content_profile.fmt(str(value))
else:
text += str(value)
text += '\n'
return text
def show_pipeline(self, dependencies, format):
report = ''
p = Profile()
for element in dependencies:
line = format
full_key, cache_key, dim_keys = element._get_full_display_key()
line = p.fmt_subst(line, 'name', element.name, fg='blue', bold=True)
line = p.fmt_subst(line, 'key', cache_key, fg='yellow', dim=dim_keys)
line = p.fmt_subst(line, 'full-key', full_key, fg='yellow', dim=dim_keys)
consistency = element._consistency()
if consistency == Consistency.INCONSISTENT:
line = p.fmt_subst(line, 'state', "no reference", fg='red')
else:
if element._cached():
line = p.fmt_subst(line, 'state', "cached", fg='magenta')
elif element._remotely_cached():
line = p.fmt_subst(line, 'state', "downloadable", fg='cyan')
elif consistency == Consistency.RESOLVED:
line = p.fmt_subst(line, 'state', "fetch needed", fg='red')
elif element._buildable():
line = p.fmt_subst(line, 'state', "buildable", fg='green')
else:
line = p.fmt_subst(line, 'state', "waiting", fg='blue')
# Element configuration
if "%{config" in format:
config = _yaml.node_sanitize(element._Element__config)
line = p.fmt_subst(
line, 'config',
yaml.round_trip_dump(config, default_flow_style=False, allow_unicode=True))
# Variables
if "%{vars" in format:
variables = _yaml.node_sanitize(element._Element__variables.variables)
line = p.fmt_subst(
line, 'vars',
yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))
# Environment
if "%{env" in format:
environment = _yaml.node_sanitize(element._Element__environment)
line = p.fmt_subst(
line, 'env',
yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
# Public
if "%{public" in format:
environment = _yaml.node_sanitize(element._Element__public)
line = p.fmt_subst(
line, 'public',
yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
# Workspaced
if "%{workspaced" in format:
line = p.fmt_subst(
line, 'workspaced',
'(workspaced)' if element._workspaced() else '', fg='yellow')
# Workspace-dirs
if "%{workspace-dirs" in format:
dirs = [path.replace(os.getenv('HOME', '/root'), '~')
for path in element._workspace_dirs()]
if len(dirs) > 0:
line = p.fmt_subst(
line, 'workspace-dirs', "Workspaces: " + ", ".join(dirs))
else:
line = p.fmt_subst(
line, 'workspace-dirs', '')
report += line + '\n'
return report.rstrip('\n')