summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorJavier Jardón <jjardon@gnome.org>2017-11-26 23:39:48 +0000
committerJavier Jardón <jjardon@gnome.org>2017-12-12 15:58:23 +0000
commit7df7f3b427739ff7d69da2ba218da0124822892c (patch)
tree843c75e9bede53862ab101d6a7bcd1da15a33c55 /scripts
parent7aad5150f69da42b84994c353283db5daf8e967f (diff)
downloaddefinitions-7df7f3b427739ff7d69da2ba218da0124822892c.tar.gz
Remove all .morph files and files from the old format
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/baserock-release-metadata66
-rwxr-xr-xscripts/check-unpetrify-refs.py76
-rwxr-xr-xscripts/cycle.sh61
-rw-r--r--scripts/licensecheck.pl604
-rwxr-xr-xscripts/licensecheck.py201
-rwxr-xr-xscripts/organize-morphologies.py255
-rwxr-xr-xscripts/release-build192
-rw-r--r--scripts/release-build.test.conf6
-rwxr-xr-xscripts/release-test400
-rwxr-xr-xscripts/release-test-os526
-rwxr-xr-xscripts/release-upload473
-rw-r--r--scripts/release-upload.test.conf10
-rw-r--r--scripts/scriptslib.py156
-rwxr-xr-xscripts/yaml-jsonschema50
14 files changed, 0 insertions, 3076 deletions
diff --git a/scripts/baserock-release-metadata b/scripts/baserock-release-metadata
deleted file mode 100755
index 1b429671..00000000
--- a/scripts/baserock-release-metadata
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (C) 2017 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-
-
-'''baserock-release-metadata
-
-Generates a YAML document with some simple metadata that should be included
-in release binaries
-
-This expects to be run inside the definitions repo so that it can figure
-out the Git commit being built.
-
-'''
-
-import collections
-import subprocess
-
-
-def git_show_commit():
- return subprocess.check_output(
- ['git', 'rev-parse', 'HEAD']).decode('unicode-escape').strip()
-
-
-def buildstream_version():
- return subprocess.check_output(
- ['bst', '--version']).decode('unicode-escape').strip()
-
-
-def main():
- repo = "https://www.gitlab.com/baserock/definitions"
- commit = git_show_commit()
- build_tool = "Buildstream"
- # This marks the version of BuildStream that checked out the sysroot
- # artifact; the artifact may have pulled it from a cache though so
- # we don't know what version of BuildStream actually built anything.
- build_tool_version = buildstream_version()
-
- # Here we produce a YAML document with specific ordering and formatting
- # to ensure readability.
- print("description: |\n"
- " This sysroot was produced by the Baserock project.\n"
- " See <http://www.baserock.org/> for more information.\n")
- print("project-repo: %s" % repo)
- print("project-commit: %s" % commit)
- print()
- print("build-tool: %s" % build_tool)
- print("build-tool-version: %s" % build_tool_version)
-
-
-try:
- main()
-except RuntimeError as e:
- sys.stderr.write("%s\n" % e)
- sys.exit(1)
diff --git a/scripts/check-unpetrify-refs.py b/scripts/check-unpetrify-refs.py
deleted file mode 100755
index c70b680d..00000000
--- a/scripts/check-unpetrify-refs.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2016 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import glob
-import argparse
-import subprocess
-
-import scriptslib
-
-
-'''
-Script for checking unpetrify-refs in strata.
-
-Without args this script will check everything in strata/, or each stratum
-given on the command-line and will print on stdout whether a chunk has
-a missing or non-existent unpetrify-ref and if it fails to check the remote
-(missing repo).
-'''
-
-strata_dir = "strata"
-
-def ref_exists(remote, ref):
- output = subprocess.check_output(
- ["git", "ls-remote", remote, str(ref)],
- stderr=subprocess.STDOUT).strip()
- return True if output else False
-
-def main():
- parser = argparse.ArgumentParser(
- description="Sanity checks unpetrify-refs in Baserock strata")
- parser.add_argument("--trove-host", default="git.baserock.org",
- help="Trove host to map repo aliases to")
- parser.add_argument("strata", nargs="*", metavar="STRATA",
- help="The strata to check (checks all by default)")
- args = parser.parse_args()
-
- if args.strata:
- strata = args.strata
- else:
- strata_path = os.path.join(scriptslib.definitions_root(), strata_dir)
- strata = glob.glob("%s/*.morph" % strata_path)
-
- for stratum in strata:
- path = os.path.relpath(stratum)
- morphology = scriptslib.load_yaml_file(stratum)
- for chunk in morphology['chunks']:
- unpetrify_ref = chunk.get("unpetrify-ref")
- if not unpetrify_ref:
- print ("%s: '%s' has no unpetrify-ref!" %
- (path, chunk['name']))
- continue
- remote = scriptslib.parse_repo_alias(chunk['repo'], args.trove_host)
- try:
- if not ref_exists(remote, unpetrify_ref):
- print ("%s: unpetrify-ref for '%s' is not present on the "
- "remote (%s)!" % (path, chunk['name'], remote))
- except subprocess.CalledProcessError as e:
- print ("%s: failed to ls-remote (%s) for chunk '%s':\n%s" %
- (path, remote, chunk['name'], e.output.strip()))
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/cycle.sh b/scripts/cycle.sh
deleted file mode 100755
index c0e2aa67..00000000
--- a/scripts/cycle.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh
-# Copyright (C) 2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-usage() {
- echo "Usage: cycle.sh some-system some-cluster [newversion]"
- echo
- echo "This builds and deploys the current checked out version of"
- echo "some-system, applying it as a self-upgrade to the system you"
- echo "are working in, using configuration from some-cluster."
- echo "The upgrade is labelled TEST by default, or [newversion] if"
- echo "specified, and is set to be the default for next boot."
-}
-
-if [ -z "$1" ] || [ -z "$2" ] || [ ! -z "$4" ] ; then
- usage
- exit 1
-fi
-
-newversion=TEST
-if [ ! -z "$3" ] ; then
- newversion=$3
- if (echo "$newversion" | grep ' ' > /dev/null 2>&1) ; then
- echo 'Version label must not contain spaces.'
- exit 1
- fi
-fi
-
-if system-version-manager get-running | grep -q "^$newversion$"; then
- echo "You are currently running the $newversion system."
- echo "Maybe you want to boot into a different system version?"
- exit 1
-fi
-
-set -e
-set -v
-
-runningversion=`system-version-manager get-running`
-system-version-manager set-default $runningversion
-if system-version-manager list | grep -q "^$newversion$"; then
- system-version-manager remove $newversion
-fi
-
-morph gc
-morph build "$1"
-
-sed -i "s|^- morph: .*$|- morph: $1|" "$2"
-morph deploy --upgrade "$2" self.HOSTNAME=$(hostname) self.VERSION_LABEL=$newversion
-system-version-manager list
diff --git a/scripts/licensecheck.pl b/scripts/licensecheck.pl
deleted file mode 100644
index 5b6d0d33..00000000
--- a/scripts/licensecheck.pl
+++ /dev/null
@@ -1,604 +0,0 @@
-#!/usr/bin/perl
-# This script was originally based on the script of the same name from
-# the KDE SDK (by dfaure@kde.org)
-#
-# This version is
-# Copyright (C) 2007, 2008 Adam D. Barratt
-# Copyright (C) 2012 Francesco Poli
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program. If not, see <https://www.gnu.org/licenses/>.
-
-=head1 NAME
-
-licensecheck - simple license checker for source files
-
-=head1 SYNOPSIS
-
-B<licensecheck> B<--help>|B<--version>
-
-B<licensecheck> [B<--no-conf>] [B<--verbose>] [B<--copyright>]
-[B<-l>|B<--lines=>I<N>] [B<-i>|B<--ignore=>I<regex>] [B<-c>|B<--check=>I<regex>]
-[B<-m>|B<--machine>] [B<-r>|B<--recursive>]
-I<list of files and directories to check>
-
-=head1 DESCRIPTION
-
-B<licensecheck> attempts to determine the license that applies to each file
-passed to it, by searching the start of the file for text belonging to
-various licenses.
-
-If any of the arguments passed are directories, B<licensecheck> will add
-the files contained within to the list of files to process.
-
-=head1 OPTIONS
-
-=over 4
-
-=item B<--verbose>, B<--no-verbose>
-
-Specify whether to output the text being processed from each file before
-the corresponding license information.
-
-Default is to be quiet.
-
-=item B<-l=>I<N>, B<--lines=>I<N>
-
-Specify the number of lines of each file's header which should be parsed
-for license information. (Default is 60).
-
-=item B<-i=>I<regex>, B<--ignore=>I<regex>
-
-When processing the list of files and directories, the regular
-expression specified by this option will be used to indicate those which
-should not be considered (e.g. backup files, VCS metadata).
-
-=item B<-r>, B<--recursive>
-
-Specify that the contents of directories should be added
-recursively.
-
-=item B<-c=>I<regex>, B<--check=>I<regex>
-
-Specify a pattern against which filenames will be matched in order to
-decide which files to check the license of.
-
-The default includes common source files.
-
-=item B<--copyright>
-
-Also display copyright text found within the file
-
-=item B<-m>, B<--machine>
-
-Display the information in a machine readable way, i.e. in the form
-<file><tab><license>[<tab><copyright>] so that it can be easily sorted
-and/or filtered, e.g. with the B<awk> and B<sort> commands.
-Note that using the B<--verbose> option will kill the readability.
-
-=item B<--no-conf>, B<--noconf>
-
-Do not read any configuration files. This can only be used as the first
-option given on the command line.
-
-=back
-
-=head1 CONFIGURATION VARIABLES
-
-The two configuration files F</etc/devscripts.conf> and
-F<~/.devscripts> are sourced by a shell in that order to set
-configuration variables. Command line options can be used to override
-configuration file settings. Environment variable settings are
-ignored for this purpose. The currently recognised variables are:
-
-=over 4
-
-=item B<LICENSECHECK_VERBOSE>
-
-If this is set to I<yes>, then it is the same as the B<--verbose> command
-line parameter being used. The default is I<no>.
-
-=item B<LICENSECHECK_PARSELINES>
-
-If this is set to a positive number then the specified number of lines
-at the start of each file will be read whilst attempting to determine
-the license(s) in use. This is equivalent to the B<--lines> command line
-option.
-
-=back
-
-=head1 LICENSE
-
-This code is copyright by Adam D. Barratt <I<adam@adam-barratt.org.uk>>,
-all rights reserved; based on a script of the same name from the KDE
-SDK, which is copyright by <I<dfaure@kde.org>>.
-This program comes with ABSOLUTELY NO WARRANTY.
-You are free to redistribute this code under the terms of the GNU
-General Public License, version 2 or later.
-
-=head1 AUTHOR
-
-Adam D. Barratt <adam@adam-barratt.org.uk>
-
-=cut
-
-use strict;
-use warnings;
-use Getopt::Long qw(:config gnu_getopt);
-use File::Basename;
-
-my $progname = basename($0);
-
-# From dpkg-source
-my $default_ignore_regex = '
-# Ignore general backup files
-(?:^|/).*~$|
-# Ignore emacs recovery files
-(?:^|/)\.#.*$|
-# Ignore vi swap files
-(?:^|/)\..*\.swp$|
-# Ignore baz-style junk files or directories
-(?:^|/),,.*(?:$|/.*$)|
-# File-names that should be ignored (never directories)
-(?:^|/)(?:DEADJOE|\.cvsignore|\.arch-inventory|\.bzrignore|\.gitignore)$|
-# File or directory names that should be ignored
-(?:^|/)(?:CVS|RCS|\.pc|\.deps|\{arch\}|\.arch-ids|\.svn|\.hg|_darcs|\.git|
-\.shelf|_MTN|\.bzr(?:\.backup|tags)?)(?:$|/.*$)
-';
-
-# Take out comments and newlines
-$default_ignore_regex =~ s/^#.*$//mg;
-$default_ignore_regex =~ s/\n//sg;
-
-my $default_check_regex = '\.(c(c|pp|xx)?|h(h|pp|xx)?|f(77|90)?|go|p(l|m)|xs|sh|php|py(|x)|rb|java|js|vala|el|sc(i|e)|cs|pas|inc|dtd|xsl|mod|m|tex|mli?|(c|l)?hs)$';
-
-my $modified_conf_msg;
-
-my %OPT=(
- verbose => '',
- lines => '',
- noconf => '',
- ignore => '',
- check => '',
- recursive => 0,
- copyright => 0,
- machine => 0,
-);
-
-my $def_lines = 60;
-
-# Read configuration files and then command line
-# This is boilerplate
-
-if (@ARGV and $ARGV[0] =~ /^--no-?conf$/) {
- $modified_conf_msg = " (no configuration files read)";
- shift;
-} else {
- my @config_files = ('/etc/devscripts.conf', '~/.devscripts');
- my %config_vars = (
- 'LICENSECHECK_VERBOSE' => 'no',
- 'LICENSECHECK_PARSELINES' => $def_lines,
- );
- my %config_default = %config_vars;
-
- my $shell_cmd;
- # Set defaults
- foreach my $var (keys %config_vars) {
- $shell_cmd .= qq[$var="$config_vars{$var}";\n];
- }
- $shell_cmd .= 'for file in ' . join(" ", @config_files) . "; do\n";
- $shell_cmd .= '[ -f $file ] && . $file; done;' . "\n";
- # Read back values
- foreach my $var (keys %config_vars) { $shell_cmd .= "echo \$$var;\n" }
- my $shell_out = `/bin/bash -c '$shell_cmd'`;
- @config_vars{keys %config_vars} = split /\n/, $shell_out, -1;
-
- # Check validity
- $config_vars{'LICENSECHECK_VERBOSE'} =~ /^(yes|no)$/
- or $config_vars{'LICENSECHECK_VERBOSE'} = 'no';
- $config_vars{'LICENSECHECK_PARSELINES'} =~ /^[1-9][0-9]*$/
- or $config_vars{'LICENSECHECK_PARSELINES'} = $def_lines;
-
- foreach my $var (sort keys %config_vars) {
- if ($config_vars{$var} ne $config_default{$var}) {
- $modified_conf_msg .= " $var=$config_vars{$var}\n";
- }
- }
- $modified_conf_msg ||= " (none)\n";
- chomp $modified_conf_msg;
-
- $OPT{'verbose'} = $config_vars{'LICENSECHECK_VERBOSE'} eq 'yes' ? 1 : 0;
- $OPT{'lines'} = $config_vars{'LICENSECHECK_PARSELINES'};
-}
-
-GetOptions(\%OPT,
- "help|h",
- "check|c=s",
- "copyright",
- "ignore|i=s",
- "lines|l=i",
- "machine|m",
- "noconf|no-conf",
- "recursive|r",
- "verbose!",
- "version|v",
-) or die "Usage: $progname [options] filelist\nRun $progname --help for more details\n";
-
-$OPT{'lines'} = $def_lines if $OPT{'lines'} !~ /^[1-9][0-9]*$/;
-$OPT{'ignore'} = $default_ignore_regex if ! length $OPT{'ignore'};
-$OPT{'check'} = $default_check_regex if ! length $OPT{'check'};
-
-if ($OPT{'noconf'}) {
- fatal("--no-conf is only acceptable as the first command-line option!");
-}
-if ($OPT{'help'}) { help(); exit 0; }
-if ($OPT{'version'}) { version(); exit 0; }
-
-die "Usage: $progname [options] filelist\nRun $progname --help for more details\n" unless @ARGV;
-
-$OPT{'lines'} = $def_lines if not defined $OPT{'lines'};
-
-my @files = ();
-my @find_args = ();
-my $files_count = @ARGV;
-
-push @find_args, qw(-maxdepth 1) unless $OPT{'recursive'};
-push @find_args, qw(-follow -type f -print);
-
-while (@ARGV) {
- my $file = shift @ARGV;
-
- if (-d $file) {
- open my $FIND, '-|', 'find', $file, @find_args
- or die "$progname: couldn't exec find: $!\n";
-
- while (<$FIND>) {
- chomp;
- next unless m%$OPT{'check'}%;
- # Skip empty files
- next if (-z $_);
- push @files, $_ unless m%$OPT{'ignore'}%;
- }
- close $FIND;
- } else {
- next unless ($files_count == 1) or $file =~ m%$OPT{'check'}%;
- push @files, $file unless $file =~ m%$OPT{'ignore'}%;
- }
-}
-
-while (@files) {
- my $file = shift @files;
- my $content = '';
- my $copyright_match;
- my $copyright = '';
- my $license = '';
- my %copyrights;
-
- open (my $F, '<' ,$file) or die "Unable to access $file\n";
- while (<$F>) {
- last if ($. > $OPT{'lines'});
- $content .= $_;
- $copyright_match = parse_copyright($_);
- if ($copyright_match) {
- $copyrights{lc("$copyright_match")} = "$copyright_match";
- }
- }
- close($F);
-
- $copyright = join(" / ", reverse sort values %copyrights);
-
- print qq(----- $file header -----\n$content----- end header -----\n\n)
- if $OPT{'verbose'};
-
- $license = parselicense(clean_comments($content));
-
- if ($OPT{'machine'}) {
- print "$file\t$license";
- print "\t" . ($copyright or "*No copyright*") if $OPT{'copyright'};
- print "\n";
- } else {
- print "$file: ";
- print "*No copyright* " unless $copyright;
- print $license . "\n";
- print " [Copyright: " . $copyright . "]\n"
- if $copyright and $OPT{'copyright'};
- print "\n" if $OPT{'copyright'};
- }
-}
-
-sub parse_copyright {
- my $copyright = '';
- my $match;
-
- my $copyright_indicator_regex = '
- (?:copyright # The full word
- |copr\. # Legally-valid abbreviation
- |\x{00a9} # Unicode character COPYRIGHT SIGN
- |\xc2\xa9 # Unicode copyright sign encoded in iso8859
- |\(c\) # Legally-null representation of sign
- )';
- my $copyright_disindicator_regex = '
- \b(?:info(?:rmation)? # Discussing copyright information
- |(notice|statement|claim|string)s? # Discussing the notice
- |and|or|is|in|to # Part of a sentence
- |(holder|owner)s? # Part of a sentence
- |ownership # Part of a sentence
- )\b';
- my $copyright_predisindicator_regex = '(
- ^[#]define\s+.*\(c\) # #define foo(c) -- not copyright
- )';
-
- if ( ! m%$copyright_predisindicator_regex%ix) {
-
- if (m%$copyright_indicator_regex(?::\s*|\s+)(\S.*)$%ix) {
- $match = $1;
-
- # Ignore lines matching "see foo for copyright information" etc.
- if ($match !~ m%^\s*$copyright_disindicator_regex%ix) {
- # De-cruft
- $match =~ s/([,.])?\s*$//;
- $match =~ s/$copyright_indicator_regex//igx;
- $match =~ s/^\s+//;
- $match =~ s/\s{2,}/ /g;
- $match =~ s/\\@/@/g;
- $copyright = $match;
- }
- }
- }
-
- return $copyright;
-}
-
-sub clean_comments {
- local $_ = shift or return q{};
-
- # Remove generic comments: look for 4 or more lines beginning with
- # regular comment pattern and trim it. Fall back to old algorithm
- # if no such pattern found.
- my @matches = m/^\s*([^a-zA-Z0-9\s]{1,3})\s\w/mg;
- if (@matches >= 4) {
- my $comment_re = qr/\s*[\Q$matches[0]\E]{1,3}\s*/;
- s/^$comment_re//mg;
- }
-
- # Remove Fortran comments
- s/^[cC] //gm;
- tr/\t\r\n/ /;
-
- # Remove C / C++ comments
- s#(\*/|/[/*])##g;
- tr% A-Za-z.,@;0-9\(\)/-%%cd;
- tr/ //s;
-
- return $_;
-}
-
-sub help {
- print <<"EOF";
-Usage: $progname [options] filename [filename ...]
-Valid options are:
- --help, -h Display this message
- --version, -v Display version and copyright info
- --no-conf, --noconf Don't read devscripts config files; must be
- the first option given
- --verbose Display the header of each file before its
- license information
- --lines, -l Specify how many lines of the file header
- should be parsed for license information
- (Default: $def_lines)
- --check, -c Specify a pattern indicating which files should
- be checked
- (Default: '$default_check_regex')
- --machine, -m Display in a machine readable way (good for awk)
- --recursive, -r Add the contents of directories recursively
- --copyright Also display the file's copyright
- --ignore, -i Specify that files / directories matching the
- regular expression should be ignored when
- checking files
- (Default: '$default_ignore_regex')
-
-Default settings modified by devscripts configuration files:
-$modified_conf_msg
-EOF
-}
-
-sub version {
- print <<"EOF";
-This is $progname, from the Debian devscripts package, version ###VERSION###
-Copyright (C) 2007, 2008 by Adam D. Barratt <adam\@adam-barratt.org.uk>; based
-on a script of the same name from the KDE SDK by <dfaure\@kde.org>.
-
-This program comes with ABSOLUTELY NO WARRANTY.
-You are free to redistribute this code under the terms of the
-GNU General Public License, version 2, or (at your option) any
-later version.
-EOF
-}
-
-sub parselicense {
- my ($licensetext) = @_;
-
- my $gplver = "";
- my $extrainfo = "";
- my $license = "";
-
- if ($licensetext =~ /version ([^, ]+?)[.,]? (?:\(?only\)?.? )?(?:of the GNU (Affero )?(Lesser |Library )?General Public License )?(as )?published by the Free Software Foundation/i or
- $licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License (?:as )?published by the Free Software Foundation[;,] version ([^, ]+?)[.,]? /i) {
-
- $gplver = " (v$1)";
- } elsif ($licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License, version (\d+(?:\.\d+)?)[ \.]/) {
- $gplver = " (v$1)";
- } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) any later version/) {
- $gplver = " (v$1 or later)";
- } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) version (\d(?:[\.-]\d+)*)/) {
- $gplver = " (v$1 or v$2)";
- }
-
- if ($licensetext =~ /(?:675 Mass Ave|59 Temple Place|51 Franklin Steet|02139|02111-1307)/i) {
- $extrainfo = " (with incorrect FSF address)$extrainfo";
- }
-
- if ($licensetext =~ /permission (?:is (also granted|given))? to link (the code of )?this program with (any edition of )?(Qt|the Qt library)/i) {
- $extrainfo = " (with Qt exception)$extrainfo"
- }
-
- if ($licensetext =~ /(All changes made in this file will be lost|DO NOT (EDIT|delete this file)|Generated (automatically|by|from)|generated.*file)/i) {
- $license = "GENERATED FILE";
- }
-
- if ($licensetext =~ /((is free software.? )?you can redistribute (it|them) and\/or modify (it|them)|is licensed) under the terms of (version [^ ]+ of )?the (GNU (Library |Lesser )General Public License|LGPL)/i) {
- $license = "LGPL$gplver$extrainfo $license";
- }
-
- if ($licensetext =~ /is free software.? you can redistribute (it|them) and\/or modify (it|them) under the terms of the (GNU Affero General Public License|AGPL)/i) {
- $license = "AGPL$gplver$extrainfo $license";
- }
-
- if ($licensetext =~ /(is free software.? )?you (can|may) redistribute (it|them) and\/or modify (it|them) under the terms of (?:version [^ ]+ (?:\(?only\)? )?of )?the GNU General Public License/i) {
- $license = "GPL$gplver$extrainfo $license";
- }
-
- if ($licensetext =~ /is distributed under the terms of the GNU General Public License,/
- and length $gplver) {
- $license = "GPL$gplver$extrainfo $license";
- }
-
- if ($licensetext =~ /is distributed.*terms.*GPL/) {
- $license = "GPL (unversioned/unknown version) $license";
- }
-
- if ($licensetext =~ /This file is part of the .*Qt GUI Toolkit. This file may be distributed under the terms of the Q Public License as defined/) {
- $license = "QPL (part of Qt) $license";
- } elsif ($licensetext =~ /may (be distributed|redistribute it) under the terms of the Q Public License/) {
- $license = "QPL $license";
- }
-
- if ($licensetext =~ /opensource\.org\/licenses\/mit-license\.php/) {
- $license = "MIT/X11 (BSD like) $license";
- } elsif ($licensetext =~ /Permission is hereby granted, free of charge, to any person obtaining a copy of this software and(\/or)? associated documentation files \(the (Software|Materials)\), to deal in the (Software|Materials)/) {
- $license = "MIT/X11 (BSD like) $license";
- } elsif ($licensetext =~ /Permission is hereby granted, without written agreement and without license or royalty fees, to use, copy, modify, and distribute this software and its documentation for any purpose/) {
- $license = "MIT/X11 (BSD like) $license";
- }
-
- if ($licensetext =~ /Permission to use, copy, modify, and(\/or)? distribute this software for any purpose with or without fee is hereby granted, provided.*copyright notice.*permission notice.*all copies/) {
- $license = "ISC $license";
- }
-
- if ($licensetext =~ /THIS SOFTWARE IS PROVIDED .*AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY/) {
- if ($licensetext =~ /All advertising materials mentioning features or use of this software must display the following acknowledge?ment.*This product includes software developed by/i) {
- $license = "BSD (4 clause) $license";
- } elsif ($licensetext =~ /(The name(?:\(s\))? .*? may not|Neither the (names? .*?|authors?) nor the names of( (its|their|other|any))? contributors may) be used to endorse or promote products derived from this software/i) {
- $license = "BSD (3 clause) $license";
- } elsif ($licensetext =~ /Redistributions of source code must retain the above copyright notice/i) {
- $license = "BSD (2 clause) $license";
- } else {
- $license = "BSD $license";
- }
- }
-
- if ($licensetext =~ /Mozilla Public License,? (Version|v\.) (\d+(?:\.\d+)?)/) {
- $license = "MPL (v$2) $license";
- }
-
- if ($licensetext =~ /Released under the terms of the Artistic License ([^ ]+)/) {
- $license = "Artistic (v$1) $license";
- }
-
- if ($licensetext =~ /is free software under the Artistic [Ll]icense/) {
- $license = "Artistic $license";
- }
-
- if ($licensetext =~ /This program is free software; you can redistribute it and\/or modify it under the same terms as Perl itself/) {
- $license = "Perl $license";
- }
-
- if ($licensetext =~ /under the Apache License, Version ([^ ]+)/) {
- $license = "Apache (v$1) $license";
- }
-
- if ($licensetext =~ /(THE BEER-WARE LICENSE)/i) {
- $license = "Beerware $license";
- }
-
- if ($licensetext =~ /This source file is subject to version ([^ ]+) of the PHP license/) {
- $license = "PHP (v$1) $license";
- }
-
- if ($licensetext =~ /under the terms of the CeCILL /) {
- $license = "CeCILL $license";
- }
-
- if ($licensetext =~ /under the terms of the CeCILL-([^ ]+) /) {
- $license = "CeCILL-$1 $license";
- }
-
- if ($licensetext =~ /under the SGI Free Software License B/) {
- $license = "SGI Free Software License B $license";
- }
-
- if ($licensetext =~ /is in the public domain/i) {
- $license = "Public domain $license";
- }
-
- if ($licensetext =~ /terms of the Common Development and Distribution License(, Version ([^(]+))? \(the License\)/) {
- $license = "CDDL " . ($1 ? "(v$2) " : '') . $license;
- }
-
- if ($licensetext =~ /Microsoft Permissive License \(Ms-PL\)/) {
- $license = "Ms-PL $license";
- }
-
- if ($licensetext =~ /Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by this license \(the \"Software\"\)/ or
- $licensetext =~ /Boost Software License([ ,-]+Version ([^ ]+)?(\.))/i) {
- $license = "BSL " . ($1 ? "(v$2) " : '') . $license;
- }
-
- if ($licensetext =~ /PYTHON SOFTWARE FOUNDATION LICENSE (VERSION ([^ ]+))/i) {
- $license = "PSF " . ($1 ? "(v$2) " : '') . $license;
- }
-
- if ($licensetext =~ /The origin of this software must not be misrepresented.*Altered source versions must be plainly marked as such.*This notice may not be removed or altered from any source distribution/ or
- $licensetext =~ /see copyright notice in zlib\.h/) {
- $license = "zlib/libpng $license";
- } elsif ($licensetext =~ /This code is released under the libpng license/) {
- $license = "libpng $license";
- }
-
- if ($licensetext =~ /Do What The Fuck You Want To Public License, Version ([^, ]+)/i) {
- $license = "WTFPL (v$1) $license";
- }
-
- if ($licensetext =~ /Do what The Fuck You Want To Public License/i) {
- $license = "WTFPL $license";
- }
-
- if ($licensetext =~ /(License WTFPL|Under (the|a) WTFPL)/i) {
- $license = "WTFPL $license";
- }
-
- $license = "UNKNOWN" if (!length($license));
-
- # Remove trailing spaces.
- $license =~ s/\s+$//;
-
- return $license;
-}
-
-sub fatal {
- my ($pack,$file,$line);
- ($pack,$file,$line) = caller();
- (my $msg = "$progname: fatal error at line $line:\n@_\n") =~ tr/\0//d;
- $msg =~ s/\n\n$/\n/;
- die $msg;
-}
diff --git a/scripts/licensecheck.py b/scripts/licensecheck.py
deleted file mode 100755
index 08d0e1b4..00000000
--- a/scripts/licensecheck.py
+++ /dev/null
@@ -1,201 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2016 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import argparse
-import errno
-import os
-import pipes
-import re
-import string
-import subprocess
-import sys
-import tempfile
-
-import scriptslib
-
-
-gpl3_chunks = ("autoconf",
- "autoconf-tarball",
- "automake",
- "bash",
- "binutils",
- "bison",
- "ccache",
- "cmake",
- "flex",
- "gawk",
- "gcc",
- "gdbm",
- "gettext-tarball",
- "gperf",
- "groff",
- "libtool",
- "libtool-tarball",
- "m4-tarball",
- "make",
- "nano",
- "patch",
- "rsync",
- "texinfo-tarball")
-
-
-def license_file_name(repo_name, sha, licenses_dir):
- license_file = os.path.join(licenses_dir, repo_name + '-' + sha)
- return license_file
-
-
-def check_license(repo_name, sha, clone_path, license_file):
-
- licenses_dir = os.path.dirname(license_file)
-
- try:
- os.makedirs(licenses_dir)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
-
- _, license_file_temp = tempfile.mkstemp(dir=licenses_dir)
- with open(license_file_temp,"wb") as out:
- sys.stderr.write("Checking license of '%s' ...\n" % repo_name)
- with open(os.devnull, 'w') as devnull:
- subprocess.check_call("perl scripts/licensecheck.pl -r "
- + pipes.quote(clone_path) + "|cut -d: -f2- | sort -u",
- stdout=out, stderr=devnull, shell=True)
-
- os.rename(license_file_temp, license_file)
- return license_file
-
-
-def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir):
- repo_name = re.split('/|:',repo)[-1]
- if repo_name.endswith(".git"):
- repo_name = repo_name[:-4]
-
- repo_url = scriptslib.parse_repo_alias(repo)
-
- # Check if ref is sha1 to speedup
- if len(ref) == 40 and all(c in string.hexdigits for c in ref):
- license_file = license_file_name(repo_name, ref, licenses_dir)
- if os.path.isfile(license_file):
- return (repo, license_file)
-
- clone_path = os.path.join(repos_dir, repo_name)
-
- if os.path.isdir(clone_path):
- sys.stderr.write("Updating repo '%s' ...\n" % repo_name)
- with open(os.devnull, 'w') as devnull:
- subprocess.check_call([
- "git", "remote", "update", "origin", "--prune"],
- stderr=devnull, stdout=devnull, cwd=clone_path)
- # Update submodules
- subprocess.check_call(
- ["git", "submodule", "update", "--recursive"],
- stderr=devnull, stdout=devnull, cwd=clone_path)
- subprocess.check_call(["git", "checkout", ref], stderr=devnull,
- stdout=devnull, cwd=clone_path)
- else:
- sys.stderr.write("Getting repo '%s' ...\n" % repo_name)
- with open(os.devnull, 'w') as devnull:
- try:
- # Attempt to use morph to obtain a repository, from morph's
- # existing local git cache if possible
- subprocess.check_call(
- ["morph", "get-repo", name, clone_path],
- stdout=devnull, stderr=devnull)
-
- except (OSError, subprocess.CalledProcessError):
- # Fall back to git clone, when morph hasn't been found on the
- # system, or otherwise fails to get a repo. This is required
- # where morph isn't available, e.g. when using YBD to build.
- # YBD currently doesn't offer a similar 'get-repo' feature.
- sys.stderr.write("Falling back to git clone.\n")
- subprocess.check_call(
- ["git", "clone", "--recursive", repo_url, clone_path],
- stdout=devnull, stderr=devnull) # also clone submodules
-
- sha = subprocess.check_output(
- ["git", "rev-parse", "HEAD"], cwd=clone_path).strip()
-
- license_file = license_file_name(repo_name, sha, licenses_dir)
- if os.path.isfile(license_file):
- return (repo, license_file)
-
- return (repo, check_license(repo_name, sha, clone_path, license_file))
-
-
-def check_stratum(stratum_file, repos_dir, licenses_dir):
- stratum = scriptslib.load_yaml_file(stratum_file)
- license_files = []
- for chunk in stratum['chunks']:
-
- name = chunk["name"]
- build_mode = chunk.get("build-mode") # Allowed to be None
-
- # Don't include bootstrap chunks and stripped gplv3 chunks
- if name in gpl3_chunks or build_mode == "bootstrap":
- continue
- repo = chunk["repo"]
- ref = chunk["ref"]
- yield check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir)
-
-
-def main():
-
- parser = argparse.ArgumentParser(
- description='Checks licenses of the components of a given System.')
- parser.add_argument('system', metavar='SYSTEM', type=str,
- help='System to check for licenses')
- parser.add_argument('--repos-dir', default="./repos",
- help='DIR to clone all the repos (default ./repos)')
- parser.add_argument('--licenses-dir', default="./licenses",
- help='DIR to store chunk license files (default ./licenses)')
-
- args = parser.parse_args()
-
- if not os.path.exists(args.repos_dir):
- os.makedirs(args.repos_dir)
-
- system = scriptslib.load_yaml_file(args.system)
- license_files = []
- for stratum in system['strata']:
- stratum_file = stratum['morph']
- stratum_path = os.path.join(scriptslib.definitions_root(), stratum_file)
- license_files.extend(check_stratum(stratum_path, args.repos_dir, args.licenses_dir))
-
- for chunk_repo, chunk_license in license_files:
- try:
- # Print repo name
- sys.stdout.write("%s\n%s\n" % (chunk_repo, '-' * len(chunk_repo)))
-
- # Print license file of the repo
- with open(chunk_license, 'r') as f:
- for line in f:
- sys.stdout.write(line)
- sys.stdout.write("\n")
- except IOError:
- # stdout is closed, no point in continuing
- # Attempt to close them explicitly to prevent cleanup problems:
- try:
- sys.stdout.flush()
- sys.stdout.close()
- except IOError:
- pass
- finally:
- exit()
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/organize-morphologies.py b/scripts/organize-morphologies.py
deleted file mode 100755
index 3072c8f8..00000000
--- a/scripts/organize-morphologies.py
+++ /dev/null
@@ -1,255 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2014-2016 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import json
-import morphlib
-import os
-import subprocess
-import sys
-import urllib
-import urllib2
-import urlparse
-import yaml
-import re
-import errno
-
-''' organize-morphologies.py:
-Tool for organizing morphologies in definitions.
-
-This script will move:
- - cluster morphologies into clusters directory
- - system morphologies into systems directory
- - stratum morphologies into strata directory
-
-This script will download the chunk morphologies for every stratum
-and placed into strata/stratum_which_the_chunk_belongs_to directory.
-
-It also modifies the morphologies fields which points to some morpholgy
-which has been moved.
-'''
-
-
-def make_request(path):
- server_url = 'http://git.baserock.org:8080/'
- url = urlparse.urljoin(server_url, '/1.0/%s' % path)
- handle = urllib2.urlopen(url)
- return handle.read()
-
-def quote(*args):
- return tuple(urllib.quote(string) for string in args)
-
-def cat_file(repo, ref, filename):
- return make_request('files?repo=%s&ref=%s&filename=%s' %
- quote(repo, ref, filename))
-
-# NOTE: This function reimplement part of morphlib's loader
-def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'):
- '''This function receives the name or the morph field of one morphology
- and returns the path of the morphology depending on the name, kind and
- if it belongs to other morphologies.
- '''
- # Dictionary which match morphology's kind and morphology's
- # directory in definitions.git
- morph_dir = { 'chunk': 'chunks', 'stratum': 'strata',
- 'system':'systems', 'cluster': 'clusters'}
- # For chunks morphologies we need to know to which stratums
- # belongs this chunk.
- if morph_kind == 'chunk':
- if belongs_to == '':
- raise morphlib.Error('Chunk morphologies need the stratum name'
- 'to create the path. Please add the stratum'
- 'which belongs this morphology')
- # Get the name of the chunk which we assume is at the end
- # of the morph file
- if '/' in morph_field:
- morph_field = os.path.basename(morph_field)
-
- # Add the stratum name to the chunk name
- morph_field = os.path.join(belongs_to, morph_field)
-
- # Reset the kind to stratum because chunk contains stratum
- # name in its path.
- morph_kind = 'stratum'
-
- # Add the morphology path to the morph field.
- if not morph_field.startswith(morph_dir[morph_kind]):
- morph_field = os.path.join(morph_dir[morph_kind], morph_field)
-
- # Add the morphology suffix if the morphology.
- if not morph_field.endswith('.morph'):
- morph_field = morph_field + '.morph'
-
- return morph_field
-
-def create_directory(name, path):
- directory = os.path.join(path, name)
- try:
- os.makedirs(directory)
- except OSError as err:
- if err.errno != errno.EEXIST:
- raise err
- else:
- pass
- return directory
-
-def move_file(morph, directory, path, loader):
- if not morph.filename.startswith(directory):
- filename = os.path.basename(morph.filename)
- new_location = os.path.join(path, filename)
- print '\nMoving %s into %s' % (filename, new_location)
- subprocess.call(['git', 'mv', morph.filename, new_location])
- morph.filename = new_location
- loader.unset_defaults(morph)
- loader.save_to_file(morph.filename, morph)
-
-def load_and_fix_chunk(chunk_str, loader, name):
- try:
- chunk_morph = loader.load_from_string(chunk_str)
- except morphlib.morphloader.InvalidFieldError as err:
- if "comments" in str(err):
- # This error is caused because there are old morphologies which
- # contain the field "comments" instead of "description".
- # Replacing "comments" field by "description" will allow the morphology
- # to pass parse_morphology_text check and ready to be written to a file.
- fixed_chunk = loader.parse_morphology_text(chunk_str, name)
- fixed_chunk['description'] = fixed_chunk.pop('comments')
- print "WARNING: Invalid 'comments' field in " \
- "%s corrected to 'description'" % name
- chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name)
- elif "buildsystem" in str(err):
- # This error is caused because a typo in a morphology which
- # has a field "buildsystem" instead of "build-system".
- fixed_chunk = loader.parse_morphology_text(chunk_str, name)
- fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem')
- print "WARNING: Invalid 'buildsystem' field in %s" \
- "corrected to 'build-system'" % name
- chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name)
- else:
- print "ERROR: %s in chunk %s" %(err, name)
- raise err
- except morphlib.morphloader.MorphologyNotYamlError as err:
- print "WARNING: %s in chunk %s is not valid YAML, " \
- "attempting to fix..." %(err, name)
- # This error is caused because there are old morphologies written
- # in JSON which contain '\t' characters. When try to load this
- # kind of morphologies load_from_string fails when parse_morphology_text.
- # Removing this characters will make load_from_string to load the morphology
- # and translate it into a correct yaml format.
- fixed_chunk = chunk_str.replace('\t','')
- print "INFO: %s successfully fixed" % name
- chunk_morph = load_and_fix_chunk(fixed_chunk, loader, name)
- return chunk_morph
-
-def move_clusters(morphs, path, loader):
- kind = 'system'
- directory = 'clusters'
- # Move cluster morphologies to clusters folder fixing their dependent
- # morphologies which are systems.
- full_path = create_directory(directory, path)
- for morph in morphs:
- all_systems = morph['systems'][:]
- for system in morph['systems']:
- all_systems.extend(system.get('subsystems', []))
- # Add the correct path to the morph fields for systems and subsystems
- for field in all_systems:
- field['morph'] = sanitise_morphology_path(field['morph'], kind)
- move_file(morph, directory, full_path, loader)
-
-def move_systems(morphs, path, loader):
- kind = 'stratum'
- directory = 'systems'
- # Move system morphologies to systems folder fixing their dependent
- # morphologies which are strata.
- full_path = create_directory(directory, path)
- for morph in morphs:
- # Add name field and the correct path to the stratum on the morph
- # fields in strata.
- for field in morph['strata']:
- field['name'] = os.path.basename(field['morph'])
- field['morph'] = sanitise_morphology_path(field['morph'], kind)
- move_file(morph, directory, full_path, loader)
-
-def download_chunks(morph, loader):
- # Download chunks morphologies defined on the stratum and
- # add them to the directory tree.
- for chunk in morph['chunks']:
- name = chunk['name'] + '.morph'
- try:
- chunk['morph'] = sanitise_morphology_path(chunk['morph'], 'chunk', morph['name'])
- except KeyError as err:
- if 'morph' in str(err):
- chunk['morph'] = sanitise_morphology_path(chunk['name'], 'chunk', morph['name'])
- else:
- raise err
- ref = chunk['ref']
- repo = scriptslib.parse_repo_alias(chunk['repo'])
- try:
- print "\nDownloading %s from %s into %s" %(name, repo, chunk['morph'])
- chunk_str = cat_file(repo, ref, name)
- except urllib2.HTTPError as err:
- # If there is no morphology in the repository we assume that the morphology
- # system will be autodetected, so we don't have to create a new one
- # unless we shut down the autodetecting system (fallback system).
- if err.code == 404:
- print 'INFO: Morph will fall-back to build-time' \
- 'autodetection for %s' %(name)
- # Remove morph field from autodetected chunks
- del chunk['morph']
- else:
- loaded_chunk = load_and_fix_chunk(chunk_str, loader, name)
- loader.unset_defaults(loaded_chunk)
- loader.save_to_file(chunk['morph'], loaded_chunk)
-
-def move_strata(morphs, path, loader):
- # Create strata directory
- strata_dir = 'strata/'
- strata_path = create_directory(strata_dir, path)
- for morph in morphs:
- # Create stratum directory where downloading its chunks.
- stratum_path = strata_path + morph['name']
- stratum_dir = create_directory(stratum_path, path)
-
- # Download chunks which belongs to the stratum
- download_chunks(morph, loader)
-
- # Add to build-depends the correct path to the dependent stratum morphologies.
- for build_depends in morph['build-depends']:
- build_depends['morph'] = sanitise_morphology_path(build_depends['morph'], 'stratum')
- # Move stratum morphologies to strata
- move_file(morph, strata_dir, strata_path, loader)
-
-def main():
- # Load all morphologies in the definitions repo
- sb = morphlib.sysbranchdir.open_from_within('.')
- loader = morphlib.morphloader.MorphologyLoader()
- morphs = [m for m in sb.load_all_morphologies(loader)]
-
- # Clasify the morphologies regarding of their kind field
- morphologies = { kind: [m for m in morphs if m['kind'] == kind]
- for kind in ('chunk', 'stratum', 'system', 'cluster') }
-
- for kind, morphs in morphologies.iteritems():
- print 'There are: %d %s.\n' %(len(morphs), kind)
-
- # Get the path from definitions repo
- definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
-
- # Move the morphologies to its directories
- move_clusters(morphologies['cluster'], definitions_repo, loader)
- move_systems(morphologies['system'], definitions_repo, loader)
- move_strata(morphologies['stratum'], definitions_repo, loader)
-
-main()
diff --git a/scripts/release-build b/scripts/release-build
deleted file mode 100755
index cb62f661..00000000
--- a/scripts/release-build
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import cliapp
-import morphlib
-import os
-import subprocess
-import sys
-import time
-
-
-class Build(object):
- '''A single distbuild instance.'''
-
- def __init__(self, name, arch, app):
- self.system_name = name
-
- controller_netloc = app.controllers[arch].split(':')
- controller_args = [
- '--controller-initiator-address=%s' % controller_netloc[0],
- ]
- if len(controller_netloc) > 1:
- controller_args.append(
- '--controller-initiator-port=%s' % controller_netloc[1])
-
- self.command = ['morph', 'distbuild', '--local-changes=ignore']
- self.command += controller_args + [self.system_name]
-
- def start(self):
- self.process = subprocess.Popen(self.command)
-
- def completed(self):
- return (self.process.poll() is not None)
-
-
-class ReleaseApp(cliapp.Application):
-
- '''Cliapp app that handles distbuilding and deploying a cluster.'''
-
- def add_settings(self):
- self.settings.string_list(['controllers'],
- 'a list of distbuild controllers and their '
- 'architecture')
-
- self.settings.string(['trove-host'],
- 'hostname of Trove instance')
-
- self.settings.string(['artifact-cache-server'],
- 'server to fetch artifacts from', default=None)
-
- self.settings.string(['release-number'],
- 'Baserock version of the systems being built',
- default='yy.ww')
-
- def error(self, message):
- raise cliapp.AppException(message)
-
- def check_args(self, args):
- if len(args) == 0:
- self.error(
- "Please pass the name of the release cluster (e.g. "
- "clusters/release.morph)")
-
- if len(args) > 1:
- self.error("Too many arguments given.")
-
- def process_args(self, args):
- '''Process the command line'''
- self.controllers = {}
- controllers_list = self.settings['controllers']
-
- for item in controllers_list:
- arch, controller = item.split(':', 1)
- self.controllers[arch] = controller
-
- defs_repo = morphlib.definitions_repo.open(
- '.', search_for_root=True)
- self.loader = defs_repo.get_morphology_loader()
- self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo)
-
- self.check_args(args)
-
- cluster_name = args[0]
- cluster, cluster_path = self.load_morphology(cluster_name)
-
- builds = self.prepare_builds(cluster)
- for build in builds:
- build.start()
-
- while not all(build.completed() for build in builds):
- time.sleep(1)
-
- fail = False
- for build in builds:
- if build.process.returncode != 0:
- fail = True
- sys.stderr.write(
- 'Building failed for %s\n' % build.system_name)
- if fail:
- raise cliapp.AppException('Building of systems failed')
-
- if not os.path.exists('release'):
- os.mkdir('release')
- self.deploy_images(cluster, cluster_path)
-
- def load_morphology(self, name, kind=None):
- path = morphlib.util.sanitise_morphology_path(name)
- morph = self.loader.load_from_string(
- self.finder.read_file(path))
- if kind:
- assert morph['kind'] == kind
- return morph, path
-
- def iterate_systems(self, system_list):
- for system in system_list:
- yield system['morph']
- if 'subsystems' in system:
- for subsystem in self.iterate_systems(system['subsystems']):
- yield subsystem
-
- def prepare_builds(self, cluster):
- '''Prepare a list of builds'''
- systems = set(self.iterate_systems(cluster['systems']))
- builds = []
- for system_name in systems:
- system, _ = self.load_morphology(system_name)
- if system['arch'] in self.controllers:
- builds.append(Build(system_name, system['arch'], self))
- else:
- print("Unable to build %s: no %s distbuild available" %
- (system_name, system['arch']))
- return builds
-
- def deploy_images(self, cluster, cluster_path):
- version_label = 'baserock-%s' % self.settings['release-number']
- outputs = {}
-
- for system in cluster['systems']:
- morphology_name = system['morph']
- morphology = self.load_morphology(morphology_name)[0]
- if morphology['arch'] not in self.controllers:
- continue
-
- for deployment_name, deployment_info in system['deploy'].iteritems():
- # The release.morph cluster must specify a basename for the file,
- # of name and extension. This script knows about name, but it
- # can't find out the appropriate file extension without second
- # guessing the behaviour of write extensions.
- basename = deployment_info['location']
-
- if '/' in basename or basename.startswith(version_label):
- raise cliapp.AppException(
- 'In %s: system %s.location should be just the base name, '
- 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name))
-
- filename = os.path.join('release', '%s-%s' % (version_label, basename))
- if os.path.exists(filename):
- self.output.write('Reusing existing deployment of %s\n' % filename)
- else:
- self.output.write('Creating %s from release.morph\n' % filename)
- self.deploy_single_image(cluster_path, deployment_name, filename, version_label)
-
- def deploy_single_image(self, cluster_path, name, location, version_label):
- deploy_command = [
- 'morph', 'deploy', cluster_path, name,
- '--trove-host=%s' % self.settings['trove-host']]
- artifact_server = self.settings['artifact-cache-server']
- if artifact_server is not None:
- deploy_command.append('--artifact-cache-server=%s' % artifact_server)
- deploy_command.extend((
- '%s.location=%s' % (name, location),
- '%s.VERSION_LABEL=%s' % (name, version_label)
- ))
-
- cliapp.runcmd(deploy_command, stdout=sys.stdout)
-
-
-ReleaseApp().run()
diff --git a/scripts/release-build.test.conf b/scripts/release-build.test.conf
deleted file mode 100644
index 50083352..00000000
--- a/scripts/release-build.test.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-[config]
-trove-host = ct-mcr-1.ducie.codethink.co.uk
-controllers = x86_64:ct-mcr-1-distbuild-x86-64-majikthise-controller.dyn.ducie.codethink.co.uk,
- x86_32:ct-mcr-1-distbuild-x86-32-majikthise-controller.dyn.ducie.codethink.co.uk,
- armv7lhf:ct-mcr-1-distbuild-armv7lhf-jetson.dyn.ducie.codethink.co.uk
-release-number = 14.29
diff --git a/scripts/release-test b/scripts/release-test
deleted file mode 100755
index 4dcc6f76..00000000
--- a/scripts/release-test
+++ /dev/null
@@ -1,400 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2014 Codethink Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-'''release-test
-
-This script deploys the set of systems in the cluster morphology it is
-instructed to read, to test that they work correctly.
-
-'''
-
-import cliapp
-import os
-import pipes
-import shlex
-import shutil
-import socket
-import tempfile
-import time
-import uuid
-
-import morphlib
-
-
-class MorphologyHelper(object):
-
- def __init__(self):
- self.defs_repo = morphlib.definitions_repo.open(
- '.', search_for_root=True)
- self.loader = morphlib.morphloader.MorphologyLoader()
- self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
-
- def load_morphology(self, path):
- text = self.finder.read_file(path)
- return self.loader.load_from_string(text)
-
- @classmethod
- def iterate_systems(cls, systems_list):
- for system in systems_list:
- yield morphlib.util.sanitise_morphology_path(system['morph'])
- if 'subsystems' in system:
- for subsystem in cls.iterate_systems(system['subsystems']):
- yield subsystem
-
- def iterate_cluster_deployments(cls, cluster_morph):
- for system in cluster_morph['systems']:
- path = morphlib.util.sanitise_morphology_path(system['morph'])
- defaults = system.get('deploy-defaults', {})
- for name, options in system['deploy'].iteritems():
- config = dict(defaults)
- config.update(options)
- yield path, name, config
-
- def load_cluster_systems(self, cluster_morph):
- for system_path in set(self.iterate_systems(cluster_morph['systems'])):
- system_morph = self.load_morphology(system_path)
- yield system_path, system_morph
-
-
-class TimeoutError(cliapp.AppException):
-
- """Error to be raised when a connection waits too long"""
-
- def __init__(self, msg):
- super(TimeoutError, self).__init__(msg)
-
-
-class VMHost(object):
-
- def __init__(self, user, address, disk_path):
- self.user = user
- self.address = address
- self.disk_path = disk_path
-
- @property
- def ssh_host(self):
- return '{user}@{address}'.format(user=self.user, address=self.address)
-
- def runcmd(self, *args, **kwargs):
- cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
-
- def virsh(self, *args, **kwargs):
- self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs)
-
-
-class DeployedSystemInstance(object):
-
- def __init__(self, deployment, config, host_machine, vm_id, rootfs_path):
- self.deployment = deployment
- self.config = config
- # TODO: Stop assuming test machine can DHCP and be assigned its
- # hostname in the deployer's resolve search path.
- self.ip_address = self.config['HOSTNAME']
- self.host_machine = host_machine
- self.vm_id = vm_id
- self.rootfs_path = rootfs_path
-
- @property
- def ssh_host(self):
- # TODO: Stop assuming we ssh into test instances as root
- return 'root@{host}'.format(host=self.ip_address)
-
- def runcmd(self, argv, chdir='.', **kwargs):
- ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
- '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
- cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
- cmd += argv
- ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
- return cliapp.runcmd(ssh_cmd, **kwargs)
-
- def _wait_for_dhcp(self, timeout):
- '''Block until given hostname resolves successfully.
-
- Raises TimeoutError if the hostname has not appeared in 'timeout'
- seconds.
-
- '''
- start_time = time.time()
- while True:
- try:
- socket.gethostbyname(self.ip_address)
- return
- except socket.gaierror:
- pass
- if time.time() > start_time + timeout:
- raise TimeoutError("Host %s did not appear after %i seconds" %
- (self.ip_address, timeout))
- time.sleep(0.5)
-
- def _wait_for_ssh(self, timeout):
- """Wait until the deployed VM is responding via SSH"""
- start_time = time.time()
- while True:
- try:
- self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
- return
- except cliapp.AppException:
- # TODO: Stop assuming the ssh part of the command is what failed
- if time.time() > start_time + timeout:
- raise TimeoutError("%s sshd did not start after %i seconds"
- % (self.ip_address, timeout))
- time.sleep(0.5)
-
- def wait_until_online(self, timeout=10):
- self._wait_for_dhcp(timeout)
- self._wait_for_ssh(timeout)
-
- def delete(self):
- # Stop and remove VM
- try:
- self.host_machine.virsh('destroy', self.vm_id)
- except cliapp.AppException as e:
- # TODO: Stop assuming that destroy failed because it wasn't running
- pass
- try:
- self.host_machine.virsh('undefine', self.vm_id, '--remove-all-storage')
- except cliapp.AppException as e:
- # TODO: Stop assuming that undefine failed because it was
- # already removed
- pass
-
-
-class Deployment(object):
-
- def __init__(self, cluster_path, name, deployment_config, host_machine):
- self.cluster_path = cluster_path
- self.name = name
- self.deployment_config = deployment_config
- self.host_machine = host_machine
-
- @staticmethod
- def _ssh_host_key_exists(hostname):
- """Check if an ssh host key exists in known_hosts"""
- if not os.path.exists('/root/.ssh/known_hosts'):
- return False
- with open('/root/.ssh/known_hosts', 'r') as known_hosts:
- return any(line.startswith(hostname) for line in known_hosts)
-
- def _update_known_hosts(self):
- if not self._ssh_host_key_exists(self.host_machine.address):
- with open('/root/.ssh/known_hosts', 'a') as known_hosts:
- cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
- stdout=known_hosts)
-
- @staticmethod
- def _generate_sshkey_config(tempdir, config):
- manifest = os.path.join(tempdir, 'manifest')
- with open(manifest, 'w') as f:
- f.write('0040700 0 0 /root/.ssh\n')
- f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
- authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
- os.makedirs(os.path.dirname(authkeys))
- with open(authkeys, 'w') as auth_f:
- with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
- shutil.copyfileobj(key_f, auth_f)
-
- install_files = shlex.split(config.get('INSTALL_FILES', ''))
- install_files.append(manifest)
- yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
-
- def deploy(self):
- self._update_known_hosts()
-
- hostname = str(uuid.uuid4())
- vm_id = hostname
- image_base = self.host_machine.disk_path
- rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
- hostname=hostname)
- loc = 'kvm+ssh://{ssh_host}/{id}/{path}'.format(
- ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
-
- options = {
- 'type': 'kvm',
- 'location': loc,
- 'AUTOSTART': 'True',
- 'HOSTNAME': hostname,
- 'DISK_SIZE': '20G',
- 'RAM_SIZE': '2G',
- 'VERSION_LABEL': 'release-test',
- }
-
- tempdir = tempfile.mkdtemp()
- try:
- options.update(
- self._generate_sshkey_config(tempdir,
- self.deployment_config))
-
- args = ['morph', 'deploy', self.cluster_path, self.name]
- for k, v in options.iteritems():
- args.append('%s.%s=%s' % (self.name, k, v))
- cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
-
- config = dict(self.deployment_config)
- config.update(options)
-
- return DeployedSystemInstance(self, config, self.host_machine,
- vm_id, rootpath)
- finally:
- shutil.rmtree(tempdir)
-
-
-class ReleaseApp(cliapp.Application):
-
- """Cliapp application which handles automatic builds and tests"""
-
- def add_settings(self):
- """Add the command line options needed"""
- group_main = 'Program Options'
- self.settings.string_list(['deployment-host'],
- 'ARCH:HOST:PATH that VMs can be deployed to',
- default=None,
- group=group_main)
- self.settings.string(['trove-host'],
- 'Address of Trove for test systems to build from',
- default=None,
- group=group_main)
- self.settings.string(['trove-id'],
- 'ID of Trove for test systems to build from',
- default=None,
- group=group_main)
- self.settings.string(['build-ref-prefix'],
- 'Prefix of build branches for test systems',
- default=None,
- group=group_main)
-
- @staticmethod
- def _run_tests(instance, system_path, system_morph,
- (trove_host, trove_id, build_ref_prefix),
- morph_helper, systems):
- instance.wait_until_online()
-
- tests = []
- def baserock_build_test(instance):
- instance.runcmd(['git', 'config', '--global', 'user.name',
- 'Test Instance of %s' % instance.deployment.name])
- instance.runcmd(['git', 'config', '--global', 'user.email',
- 'ci-test@%s' % instance.config['HOSTNAME']])
- instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
- '/src/tmp'])
- def morph_cmd(*args, **kwargs):
- # TODO: decide whether to use cached artifacts or not by
- # adding --artifact-cache-server= --cache-server=
- argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
- '--tempdir=/src/tmp', '--log-max=100M',
- '--trove-host', trove_host, '--trove-id', trove_id,
- '--build-ref-prefix', build_ref_prefix]
- argv.extend(args)
- instance.runcmd(argv, **kwargs)
-
- repo = morph_helper.sb.root_repository_url
- ref = morph_helper.defs_repo.HEAD
- sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref)
- morph_cmd('init', '/src/ws')
- chdir = '/src/ws'
-
- morph_cmd('checkout', repo, ref, chdir=chdir)
- # TODO: Add a morph subcommand that gives the path to the root repository.
- repo_path = os.path.relpath(
- morph_helper.sb.get_git_directory_name(repo),
- morph_helper.sb.root_directory)
- chdir = os.path.join(chdir, ref, repo_path)
-
- instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
- print 'Building test systems for {sys}'.format(sys=system_path)
- for to_build_path, to_build_morph in systems.iteritems():
- if to_build_morph['arch'] == system_morph['arch']:
- print 'Test building {path}'.format(path=to_build_path)
- morph_cmd('build', to_build_path, chdir=chdir,
- stdin=None, stdout=None, stderr=None)
- print 'Finished Building test systems'
-
- def python_smoke_test(instance):
- instance.runcmd(['python', '-c', 'print "Hello World"'])
-
- # TODO: Come up with a better way of determining which tests to run
- if 'devel' in system_path:
- tests.append(baserock_build_test)
- else:
- tests.append(python_smoke_test)
-
- for test in tests:
- test(instance)
-
- def deploy_and_test_systems(self, cluster_path,
- deployment_hosts, build_test_config):
- """Run the deployments and tests"""
-
- version = 'release-test'
-
- morph_helper = MorphologyHelper()
- cluster_morph = morph_helper.load_morphology(cluster_path)
- systems = dict(morph_helper.load_cluster_systems(cluster_morph))
-
- for system_path, deployment_name, deployment_config in \
- morph_helper.iterate_cluster_deployments(cluster_morph):
-
- system_morph = systems[system_path]
- # We can only test systems in KVM that have a BSP
- if not any('bsp' in si['morph'] for si in system_morph['strata']):
- continue
-
- # We can only test systems in KVM that we have a host for
- if system_morph['arch'] not in deployment_hosts:
- continue
- host_machine = deployment_hosts[system_morph['arch']]
- deployment = Deployment(cluster_path, deployment_name,
- deployment_config, host_machine)
-
- instance = deployment.deploy()
- try:
- self._run_tests(instance, system_path, system_morph,
- build_test_config, morph_helper, systems)
- finally:
- instance.delete()
-
- def process_args(self, args):
- """Process the command line args and kick off the builds/tests"""
- if self.settings['build-ref-prefix'] is None:
- self.settings['build-ref-prefix'] = (
- os.path.join(self.settings['trove-id'], 'builds'))
- for setting in ('deployment-host', 'trove-host',
- 'trove-id', 'build-ref-prefix'):
- self.settings.require(setting)
-
- deployment_hosts = {}
- for host_config in self.settings['deployment-host']:
- arch, address = host_config.split(':', 1)
- user, address = address.split('@', 1)
- address, disk_path = address.split(':', 1)
- if user == '':
- user = 'root'
- # TODO: Don't assume root is the user with deploy access
- deployment_hosts[arch] = VMHost(user, address, disk_path)
-
- build_test_config = (self.settings['trove-host'],
- self.settings['trove-id'],
- self.settings['build-ref-prefix'])
-
- if len(args) != 1:
- raise cliapp.AppException('Usage: release-test CLUSTER')
- cluster_path = morphlib.util.sanitise_morphology_path(args[0])
- self.deploy_and_test_systems(cluster_path, deployment_hosts,
- build_test_config)
-
-
-if __name__ == '__main__':
- ReleaseApp().run()
diff --git a/scripts/release-test-os b/scripts/release-test-os
deleted file mode 100755
index 06e01daf..00000000
--- a/scripts/release-test-os
+++ /dev/null
@@ -1,526 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2014 Codethink Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-'''release-test
-
-This script deploys the set of systems in the cluster morphology it is
-instructed to read, to test that they work correctly.
-
-'''
-
-import cliapp
-import os
-import pipes
-import shlex
-import shutil
-import socket
-import tempfile
-import time
-import uuid
-
-import morphlib
-
-
-class NovaList:
- def __init__(self):
- self.output = []
- self.lines = []
- self.instance = []
-
- def update(self):
- self.output = cliapp.runcmd(['nova', 'list'])
- self.lines = self.output.split('\n')
- self.lines = self.lines[3:-2]
-
- def get_nova_details_for_instance(self, name):
- self.update()
-
- for line in self.lines:
- entries = line.split('|')
- stripped_line = [entry.strip() for entry in entries]
- if stripped_line.count(name) == 1:
- self.instance = stripped_line
-
- def get_nova_state_for_instance(self, name):
- self.get_nova_details_for_instance(name)
- if not self.instance:
- return
- return self.instance[3]
-
- def get_nova_ip_for_instance(self, name):
- self.get_nova_details_for_instance(name)
- if not self.instance:
- return
-
- if self.get_nova_state_for_instance(name) != 'ACTIVE':
- return
-
- return self.instance[6]
-
- def get_nova_ip_for_instance_timeout(self, name, timeout=120):
- start_time = time.time()
-
- while self.get_nova_state_for_instance(name) != 'ACTIVE':
-
- if time.time() > start_time + timeout:
- print "%s not ACTIVE after %i seconds" % (name, timeout)
- return
-
- time.sleep(1)
-
- ip_addr = self.get_nova_ip_for_instance(name)
- if not ip_addr:
- return
-
- if ip_addr.count('=') == 0:
- return
-
- ip_addr = ip_addr[ip_addr.find('=') + 1:]
-
- if ip_addr.count(',') == 0:
- return ip_addr
-
- return ip_addr[:ip_addr.find(',')]
-
-
-
-class MorphologyHelper(object):
-
- def __init__(self):
- self.sb = sb = morphlib.sysbranchdir.open_from_within('.')
- defs_repo_path = sb.get_git_directory_name(sb.root_repository_url)
- self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path)
- self.loader = morphlib.morphloader.MorphologyLoader()
- self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
-
- def load_morphology(self, path):
- text = self.finder.read_file(path)
- return self.loader.load_from_string(text)
-
- @classmethod
- def iterate_systems(cls, systems_list):
- for system in systems_list:
- yield morphlib.util.sanitise_morphology_path(system['morph'])
- if 'subsystems' in system:
- for subsystem in cls.iterate_systems(system['subsystems']):
- yield subsystem
-
- def iterate_cluster_deployments(cls, cluster_morph):
- for system in cluster_morph['systems']:
- path = morphlib.util.sanitise_morphology_path(system['morph'])
- defaults = system.get('deploy-defaults', {})
- for name, options in system['deploy'].iteritems():
- config = dict(defaults)
- config.update(options)
- yield path, name, config
-
- def load_cluster_systems(self, cluster_morph):
- for system_path in set(self.iterate_systems(cluster_morph['systems'])):
- system_morph = self.load_morphology(system_path)
- yield system_path, system_morph
-
-
-class TimeoutError(cliapp.AppException):
-
- """Error to be raised when a connection waits too long"""
-
- def __init__(self, msg):
- super(TimeoutError, self).__init__(msg)
-
-
-class VMHost(object):
-
- def __init__(self, user, address, disk_path):
- self.user = user
- self.address = address
- self.disk_path = disk_path
-
- @property
- def ssh_host(self):
- return '{user}@{address}'.format(user=self.user, address=self.address)
-
- def runcmd(self, *args, **kwargs):
- cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
-
-
-class DeployedSystemInstance(object):
-
- def __init__(self, deployment, config, host_machine, vm_id, rootfs_path,
- ip_addr, hostname):
- self.deployment = deployment
- self.config = config
- self.ip_address = ip_addr
- self.host_machine = host_machine
- self.vm_id = vm_id
- self.rootfs_path = rootfs_path
- self.hostname = hostname
-
- @property
- def ssh_host(self):
- # TODO: Stop assuming we ssh into test instances as root
- return 'root@{host}'.format(host=self.ip_address)
-
- def runcmd(self, argv, chdir='.', **kwargs):
- ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
- '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
- cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
- cmd += argv
- ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
- return cliapp.runcmd(ssh_cmd, **kwargs)
-
- def _wait_for_dhcp(self, timeout):
- '''Block until given hostname resolves successfully.
-
- Raises TimeoutError if the hostname has not appeared in 'timeout'
- seconds.
-
- '''
- start_time = time.time()
- while True:
- try:
- socket.gethostbyname(self.ip_address)
- return
- except socket.gaierror:
- pass
- if time.time() > start_time + timeout:
- raise TimeoutError("Host %s did not appear after %i seconds" %
- (self.ip_address, timeout))
- time.sleep(0.5)
-
- def _wait_for_ssh(self, timeout):
- """Wait until the deployed VM is responding via SSH"""
- start_time = time.time()
- while True:
- try:
- self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
- return
- except cliapp.AppException:
- # TODO: Stop assuming the ssh part of the command is what failed
- if time.time() > start_time + timeout:
- raise TimeoutError("%s sshd did not start after %i seconds"
- % (self.ip_address, timeout))
- time.sleep(0.5)
-
- def _wait_for_cloud_init(self, timeout):
- """Wait until cloud init has resized the disc"""
- start_time = time.time()
- while True:
- try:
- out = self.runcmd(['sh', '-c',
- 'test -e "$1" && echo exists || echo does not exist',
- '-',
- '/root/cloud-init-finished'])
- except:
- import traceback
- traceback.print_exc()
- raise
- if out.strip() == 'exists':
- return
- if time.time() > start_time + timeout:
- raise TimeoutError("Disc size not increased after %i seconds"
- % (timeout))
- time.sleep(3)
-
- def wait_until_online(self, timeout=120):
- self._wait_for_dhcp(timeout)
- self._wait_for_ssh(timeout)
- self._wait_for_cloud_init(timeout)
- print "Test system %s ready to run tests." % (self.hostname)
-
- def delete(self):
- # Stop and remove VM
- print "Deleting %s test instance" % (self.hostname)
- try:
- cliapp.runcmd(['nova', 'delete', self.hostname])
- except cliapp.AppException as e:
- # TODO: Stop assuming that delete failed because the instance
- # wasn't running
- print "- Failed"
- pass
- print "Deleting %s test disc image" % (self.hostname)
- try:
- cliapp.runcmd(['nova', 'image-delete', self.hostname])
- except cliapp.AppException as e:
- # TODO: Stop assuming that image-delete failed because it was
- # already removed
- print "- Failed"
- pass
-
-
-class Deployment(object):
-
- def __init__(self, cluster_path, name, deployment_config,
- host_machine, net_id):
- self.cluster_path = cluster_path
- self.name = name
- self.deployment_config = deployment_config
- self.host_machine = host_machine
- self.net_id = net_id
-
- @staticmethod
- def _ssh_host_key_exists(hostname):
- """Check if an ssh host key exists in known_hosts"""
- if not os.path.exists('/root/.ssh/known_hosts'):
- return False
- with open('/root/.ssh/known_hosts', 'r') as known_hosts:
- return any(line.startswith(hostname) for line in known_hosts)
-
- def _update_known_hosts(self):
- if not self._ssh_host_key_exists(self.host_machine.address):
- with open('/root/.ssh/known_hosts', 'a') as known_hosts:
- cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
- stdout=known_hosts)
-
- @staticmethod
- def _generate_sshkey_config(tempdir, config):
- manifest = os.path.join(tempdir, 'manifest')
- with open(manifest, 'w') as f:
- f.write('0040700 0 0 /root/.ssh\n')
- f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
- authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
- os.makedirs(os.path.dirname(authkeys))
- with open(authkeys, 'w') as auth_f:
- with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
- shutil.copyfileobj(key_f, auth_f)
-
- install_files = shlex.split(config.get('INSTALL_FILES', ''))
- install_files.append(manifest)
- yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
-
- def deploy(self):
- self._update_known_hosts()
-
- hostname = str(uuid.uuid4())
- vm_id = hostname
- image_base = self.host_machine.disk_path
- rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
- hostname=hostname)
- loc = 'http://{ssh_host}:5000/v2.0'.format(
- ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
-
- options = {
- 'type': 'openstack',
- 'location': loc,
- 'HOSTNAME': hostname,
- 'DISK_SIZE': '5G',
- 'RAM_SIZE': '2G',
- 'VERSION_LABEL': 'release-test',
- 'OPENSTACK_USER': os.environ['OS_USERNAME'],
- 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'],
- 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'],
- 'OPENSTACK_IMAGENAME': hostname,
- 'CLOUD_INIT': 'yes',
- 'KERNEL_ARGS': 'console=tty0 console=ttyS0',
- }
-
- tempdir = tempfile.mkdtemp()
- try:
- options.update(
- self._generate_sshkey_config(tempdir,
- self.deployment_config))
-
- # Deploy the image to openstack
- args = ['morph', 'deploy', self.cluster_path, self.name]
- for k, v in options.iteritems():
- args.append('%s.%s=%s' % (self.name, k, v))
- cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
-
- config = dict(self.deployment_config)
- config.update(options)
-
- # Boot an instance from the image
- args = ['nova', 'boot',
- '--flavor', 'm1.medium',
- '--image', hostname,
- '--user-data', '/usr/lib/mason/os-init-script',
- '--nic', "net-id=%s" % (self.net_id),
- hostname]
- output = cliapp.runcmd(args)
-
- # Print nova boot output, with adminPass line removed
- output_lines = output.split('\n')
- for line in output_lines:
- if line.find('adminPass') != -1:
- password_line = line
- output_lines.remove(password_line)
- output = '\n'.join(output_lines)
- print output
-
- # Get ip address from nova list
- nl = NovaList()
- ip_addr = nl.get_nova_ip_for_instance_timeout(hostname)
- print "IP address for instance %s: %s" % (hostname, ip_addr)
-
- return DeployedSystemInstance(self, config, self.host_machine,
- vm_id, rootpath, ip_addr, hostname)
- finally:
- shutil.rmtree(tempdir)
-
-
-class ReleaseApp(cliapp.Application):
-
- """Cliapp application which handles automatic builds and tests"""
-
- def add_settings(self):
- """Add the command line options needed"""
- group_main = 'Program Options'
- self.settings.string_list(['deployment-host'],
- 'ARCH:HOST:PATH that VMs can be deployed to',
- default=None,
- group=group_main)
- self.settings.string(['trove-host'],
- 'Address of Trove for test systems to build from',
- default=None,
- group=group_main)
- self.settings.string(['trove-id'],
- 'ID of Trove for test systems to build from',
- default=None,
- group=group_main)
- self.settings.string(['build-ref-prefix'],
- 'Prefix of build branches for test systems',
- default=None,
- group=group_main)
- self.settings.string(['net-id'],
- 'Openstack network ID',
- default=None,
- group=group_main)
-
- @staticmethod
- def _run_tests(instance, system_path, system_morph,
- (trove_host, trove_id, build_ref_prefix),
- morph_helper, systems):
- instance.wait_until_online()
-
- tests = []
- def baserock_build_test(instance):
- instance.runcmd(['git', 'config', '--global', 'user.name',
- 'Test Instance of %s' % instance.deployment.name])
- instance.runcmd(['git', 'config', '--global', 'user.email',
- 'ci-test@%s' % instance.config['HOSTNAME']])
- instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
- '/src/tmp'])
- def morph_cmd(*args, **kwargs):
- # TODO: decide whether to use cached artifacts or not by
- # adding --artifact-cache-server= --cache-server=
- argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
- '--tempdir=/src/tmp', '--log-max=100M',
- '--trove-host', trove_host, '--trove-id', trove_id,
- '--build-ref-prefix', build_ref_prefix]
- argv.extend(args)
- instance.runcmd(argv, **kwargs)
-
- repo = morph_helper.sb.root_repository_url
- ref = morph_helper.defs_repo.HEAD
- sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref)
- morph_cmd('init', '/src/ws')
- chdir = '/src/ws'
-
- morph_cmd('checkout', repo, ref, chdir=chdir)
- # TODO: Add a morph subcommand that gives the path to the root repository.
- repo_path = os.path.relpath(
- morph_helper.sb.get_git_directory_name(repo),
- morph_helper.sb.root_directory)
- chdir = os.path.join(chdir, ref, repo_path)
-
- instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
- print 'Building test systems for {sys}'.format(sys=system_path)
- for to_build_path, to_build_morph in systems.iteritems():
- if to_build_morph['arch'] == system_morph['arch']:
- print 'Test building {path}'.format(path=to_build_path)
- morph_cmd('build', to_build_path, chdir=chdir,
- stdin=None, stdout=None, stderr=None)
- print 'Finished Building test systems'
-
- def python_smoke_test(instance):
- instance.runcmd(['python', '-c', 'print "Hello World"'])
-
- # TODO: Come up with a better way of determining which tests to run
- if 'devel' in system_path:
- tests.append(baserock_build_test)
- else:
- tests.append(python_smoke_test)
-
- for test in tests:
- test(instance)
-
- def deploy_and_test_systems(self, cluster_path,
- deployment_hosts, build_test_config,
- net_id):
- """Run the deployments and tests"""
-
- version = 'release-test'
-
- morph_helper = MorphologyHelper()
- cluster_morph = morph_helper.load_morphology(cluster_path)
- systems = dict(morph_helper.load_cluster_systems(cluster_morph))
-
- for system_path, deployment_name, deployment_config in \
- morph_helper.iterate_cluster_deployments(cluster_morph):
-
- system_morph = systems[system_path]
- # We can only test systems in KVM that have a BSP
- if not any('bsp' in si['morph'] for si in system_morph['strata']):
- continue
-
- # We can only test systems in KVM that we have a host for
- if system_morph['arch'] not in deployment_hosts:
- continue
- host_machine = deployment_hosts[system_morph['arch']]
- deployment = Deployment(cluster_path, deployment_name,
- deployment_config, host_machine,
- net_id)
-
- instance = deployment.deploy()
- try:
- self._run_tests(instance, system_path, system_morph,
- build_test_config, morph_helper, systems)
- finally:
- instance.delete()
-
- def process_args(self, args):
- """Process the command line args and kick off the builds/tests"""
- if self.settings['build-ref-prefix'] is None:
- self.settings['build-ref-prefix'] = (
- os.path.join(self.settings['trove-id'], 'builds'))
- for setting in ('deployment-host', 'trove-host',
- 'trove-id', 'build-ref-prefix', 'net-id'):
- self.settings.require(setting)
-
- deployment_hosts = {}
- for host_config in self.settings['deployment-host']:
- arch, address = host_config.split(':', 1)
- user, address = address.split('@', 1)
- address, disk_path = address.split(':', 1)
- if user == '':
- user = 'root'
- # TODO: Don't assume root is the user with deploy access
- deployment_hosts[arch] = VMHost(user, address, disk_path)
-
- build_test_config = (self.settings['trove-host'],
- self.settings['trove-id'],
- self.settings['build-ref-prefix'])
-
- if len(args) != 1:
- raise cliapp.AppException('Usage: release-test CLUSTER')
- cluster_path = morphlib.util.sanitise_morphology_path(args[0])
- self.deploy_and_test_systems(cluster_path, deployment_hosts,
- build_test_config,
- self.settings['net-id'])
-
-
-if __name__ == '__main__':
- ReleaseApp().run()
diff --git a/scripts/release-upload b/scripts/release-upload
deleted file mode 100755
index f8b3337c..00000000
--- a/scripts/release-upload
+++ /dev/null
@@ -1,473 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-'''Upload and publish Baserock binaries for a release.
-
-This utility is used for the Baserock release process. See
-http://wiki.baserock.org/guides/release-process/ for details on the
-release process.
-
-This utility uploads two sets of binaries:
-
-* The build artifacts (built chunks and strata) used to construct the
- systems being released. The systems are found in `release.morph` and
- the artifacts from the Trove used to prepare the release. They get
- uploaded to a public Trove (by default git.baserock.org). If they're
- the same Trove, then nothing happens.
-
-* The released system images (disk images, tar archives, etc)
- specified in `release.morph` get uploaded to a download server (by
- default download.baserock.org).
-
-'''
-
-
-import json
-import logging
-import os
-import pwd
-import shutil
-import sys
-import urllib
-import urllib2
-import urlparse
-
-import cliapp
-import yaml
-
-import morphlib
-
-class ReleaseUploader(cliapp.Application):
-
- def add_settings(self):
- group = 'Release upload settings'
-
- local_username = self.get_local_username()
-
- self.settings.string(
- ['build-trove-host'],
- 'get build artifacts from Trove at ADDRESS',
- metavar='ADDRESS',
- group=group)
-
- self.settings.string(
- ['public-trove-host'],
- 'publish build artifacts on Trove at ADDRESS',
- metavar='ADDRESS',
- default='git.baserock.org',
- group=group)
-
- self.settings.string(
- ['public-trove-username'],
- 'log into public trove as USER',
- metavar='USER',
- default=local_username,
- group=group)
-
- self.settings.string(
- ['public-trove-artifact-dir'],
- 'put published artifacts into DIR',
- metavar='DIR',
- default='/home/cache/artifacts',
- group=group)
-
- self.settings.string(
- ['release-artifact-dir'],
- 'get release artifacts from DIR (all files from there)',
- metavar='DIR',
- default='.',
- group=group)
-
- self.settings.string(
- ['download-server-address'],
- 'publish release artifacts on server at ADDRESS',
- metavar='ADDRESS',
- default='download.baserock.org',
- group=group)
-
- self.settings.string(
- ['download-server-username'],
- 'log into download server as USER',
- metavar='USER',
- default=local_username,
- group=group)
-
- self.settings.string(
- ['download-server-private-dir'],
- 'use DIR as the temporary location for uploaded release '
- 'artifacts',
- metavar='DIR',
- default='/srv/download.baserock.org/baserock/.publish-temp',
- group=group)
-
- self.settings.string(
- ['download-server-public-dir'],
- 'put published release artifacts in DIR',
- metavar='DIR',
- default='/srv/download.baserock.org/baserock',
- group=group)
-
- self.settings.string(
- ['local-build-artifacts-dir'],
- 'keep build artifacts to be uploaded temporarily in DIR',
- metavar='DIR',
- default='build-artifacts',
- group=group)
-
- self.settings.string(
- ['morph-cmd'],
- 'run FILE to invoke morph',
- metavar='FILE',
- default='morph',
- group=group)
-
- self.settings.string_list(
- ['arch'],
- 'Upload files from morphologies of ARCH',
- metavar='ARCH',
- default=[],
- group=group)
-
- self.settings.boolean(
- ['upload-build-artifacts'],
- 'upload build artifacts?',
- default=True)
-
- self.settings.boolean(
- ['upload-release-artifacts'],
- 'upload release artifacts (disk images etc)?',
- default=True)
-
- def get_local_username(self):
- uid = os.getuid()
- return pwd.getpwuid(uid)[0]
-
- def process_args(self, args):
- if len(args) != 1:
- raise cliapp.AppException('Usage: release-upload CLUSTER')
- cluster_morphology_path = args[0]
- self.status(msg='Uploading and publishing Baserock release')
-
- if self.settings['upload-build-artifacts']:
- self.publish_build_artifacts(cluster_morphology_path)
- else:
- self.status(
- msg='Not uploading build artifacts '
- '(upload-build-artifacts set to false')
-
- if self.settings['upload-release-artifacts']:
- self.publish_release_artifacts()
- else:
- self.status(
- msg='Not uploading release artifacts '
- '(upload-release-artifacts set to false')
-
- def publish_build_artifacts(self, cluster_morphology_path):
- publisher = BuildArtifactPublisher(self.settings, self.status)
- publisher.publish_build_artifacts(cluster_morphology_path)
- self.status(msg='Build artifacts have been published')
-
- def publish_release_artifacts(self):
- publisher = ReleaseArtifactPublisher(self.settings, self.status)
- publisher.publish_release_artifacts()
- self.status(msg='Release artifacts have been published')
-
- def status(self, msg, **kwargs):
- formatted = msg.format(**kwargs)
- logging.info(formatted)
- sys.stdout.write(formatted + '\n')
- sys.stdout.flush()
-
-
-class BuildArtifactPublisher(object):
-
- '''Publish build artifacts related to the release.'''
-
- def __init__(self, settings, status):
- self.settings = settings
- self.status = status
-
- def publish_build_artifacts(self, cluster_path):
- artifact_basenames = self.list_build_artifacts_for_release(cluster_path)
- self.status(
- msg='Found {count} build artifact files in release',
- count=len(artifact_basenames))
-
- to_be_uploaded = self.filter_away_build_artifacts_on_public_trove(
- artifact_basenames)
-
- logging.debug('List of artifacts (basenames) to upload (without already uploaded):')
- for i, basename in enumerate(to_be_uploaded):
- logging.debug(' {0}: {1}'.format(i, basename))
- logging.debug('End of artifact list (to_be_uploaded)')
-
- self.status(
- msg='Need to fetch locally, then upload {count} build artifacts',
- count=len(to_be_uploaded))
-
- self.upload_build_artifacts_to_public_trove(to_be_uploaded)
-
- def list_build_artifacts_for_release(self, cluster_morphology_path):
- self.status(msg='Find build artifacts included in release')
-
- # FIXME: These are hardcoded for simplicity. They would be
- # possible to deduce automatically from the workspace, but
- # that can happen later.
- repo = 'file://%s' % os.path.abspath('.')
- ref = 'HEAD'
-
- argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet',
- '--repo', repo, '--ref', ref]
- argv += self.find_system_morphologies(cluster_morphology_path)
- output = cliapp.runcmd(argv)
- basenames = output.splitlines()
- logging.debug('List of build artifacts in release:')
- for basename in basenames:
- logging.debug(' {0}'.format(basename))
- logging.debug('End of list of build artifacts in release')
-
- return basenames
-
- def find_system_morphologies(self, cluster_morphology_path):
- cluster = self.load_cluster_morphology(cluster_morphology_path)
- system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster)
- if self.settings['arch']:
- system_dicts = self.choose_systems_for_wanted_architectures(
- system_dicts, self.settings['arch'])
- return [sd['morph'] for sd in system_dicts]
-
- def load_cluster_morphology(self, pathname):
- with open(pathname) as f:
- return yaml.load(f)
-
- def find_systems_in_parsed_cluster_morphology(self, cluster):
- return cluster['systems']
-
- def choose_systems_for_wanted_architectures(self, system_dicts, archs):
- return [
- sd
- for sd in system_dicts
- if self.system_is_for_wanted_arch(sd, archs)]
-
- def system_is_for_wanted_arch(self, system_dict, archs):
- morph = self.load_system_morphology(system_dict)
- return morph['arch'] in archs
-
- def load_system_morphology(self, system_dict):
- pathname = morphlib.util.sanitise_morphology_path(system_dict['morph'])
- return self.load_morphology_from_named_file(pathname)
-
- def load_morphology_from_named_file(self, pathname):
- finder = self.get_morphology_finder_for_root_repository()
- morphology_text = finder.read_file(pathname)
- loader = morphlib.morphloader.MorphologyLoader()
- return loader.load_from_string(morphology_text)
-
- def get_morphology_finder_for_root_repository(self):
- definitions_repo = morphlib.definitions_repo.open(
- '.', search_for_root=True)
- return morphlib.morphologyfinder.MorphologyFinder(definitions_repo)
-
- def filter_away_build_artifacts_on_public_trove(self, basenames):
- result = []
- logging.debug('Filtering away already existing artifacts:')
- for basename, exists in self.query_public_trove_for_artifacts(basenames):
- logging.debug(' {0}: {1}'.format(basename, exists))
- if not exists:
- result.append(basename)
- logging.debug('End of filtering away')
- return result
-
- def query_public_trove_for_artifacts(self, basenames):
- host = self.settings['public-trove-host']
-
- # FIXME: This could use
- # contextlib.closing(urllib2.urlopen(url, data=data) instead
- # of explicit closing.
- url = 'http://{host}:8080/1.0/artifacts'.format(host=host)
- data = json.dumps(basenames)
- f = urllib2.urlopen(url, data=data)
- obj = json.load(f)
- return obj.items()
-
- def upload_build_artifacts_to_public_trove(self, basenames):
- self.download_artifacts_locally(basenames)
- self.upload_artifacts_to_public_trove(basenames)
-
- def download_artifacts_locally(self, basenames):
- dirname = self.settings['local-build-artifacts-dir']
- self.create_directory_if_missing(dirname)
- for i, basename in enumerate(basenames):
- url = self.construct_artifact_url(basename)
- pathname = os.path.join(dirname, basename)
- if not os.path.exists(pathname):
- self.status(
- msg='Downloading {i}/{total} {basename}',
- basename=repr(basename), i=i, total=len(basenames))
- self.download_from_url(url, dirname, pathname)
-
- def create_directory_if_missing(self, dirname):
- if not os.path.exists(dirname):
- os.makedirs(dirname)
-
- def construct_artifact_url(self, basename):
- scheme = 'http'
- netloc = '{host}:8080'.format(host=self.settings['build-trove-host'])
- path = '/1.0/artifacts'
- query = 'filename={0}'.format(urllib.quote_plus(basename))
- fragment = ''
- components = (scheme, netloc, path, query, fragment)
- return urlparse.urlunsplit(components)
-
- def download_from_url(self, url, dirname, pathname):
- logging.info(
- 'Downloading {url} to {pathname}'.format(
- url=url, pathname=pathname))
- with open(pathname, 'wb') as output:
- try:
- incoming = urllib2.urlopen(url)
- shutil.copyfileobj(incoming, output)
- incoming.close()
- except urllib2.HTTPError as e:
- if pathname.endswith('.meta'):
- return
- self.status(
- msg="ERROR: Can't download {url}: {explanation}",
- url=url,
- explanation=str(e))
- os.remove(pathname)
- raise
-
- def upload_artifacts_to_public_trove(self, basenames):
- self.status(
- msg='Upload build artifacts to {trove}',
- trove=self.settings['public-trove-host'])
- rsync_files_to_server(
- self.settings['local-build-artifacts-dir'],
- basenames,
- self.settings['public-trove-username'],
- self.settings['public-trove-host'],
- self.settings['public-trove-artifact-dir'])
- set_permissions_on_server(
- self.settings['public-trove-username'],
- self.settings['public-trove-host'],
- self.settings['public-trove-artifact-dir'],
- basenames)
-
-class ReleaseArtifactPublisher(object):
-
- '''Publish release artifacts for a release.'''
-
- def __init__(self, settings, status):
- self.settings = settings
- self.status = status
-
- def publish_release_artifacts(self):
- files = self.list_release_artifacts()
- if files:
- self.upload_release_artifacts_to_private_dir(files)
- self.move_release_artifacts_to_public_dir(files)
- self.create_symlinks_to_new_release_artifacts(files)
-
- def list_release_artifacts(self):
- self.status(msg='Find release artifacts to publish')
- return os.listdir(self.settings['release-artifact-dir'])
-
- def upload_release_artifacts_to_private_dir(self, files):
- self.status(msg='Upload release artifacts to private directory')
- path = self.settings['download-server-private-dir']
- self.create_directory_on_download_server(path)
- self.rsync_files_to_download_server(files, path)
-
- def create_directory_on_download_server(self, path):
- user = self.settings['download-server-username']
- host = self.settings['download-server-address']
- self.status(msg='Create {host}:{path}', host=host, path=path)
- target = '{user}@{host}'.format(user=user, host=host)
- cliapp.ssh_runcmd(target, ['mkdir', '-p', path])
-
- def rsync_files_to_download_server(self, files, path):
- self.status(msg='Upload release artifacts to download server')
- rsync_files_to_server(
- self.settings['release-artifact-dir'],
- files,
- self.settings['download-server-username'],
- self.settings['download-server-address'],
- path)
- set_permissions_on_server(
- self.settings['download-server-username'],
- self.settings['download-server-address'],
- path,
- files)
-
- def move_release_artifacts_to_public_dir(self, files):
- self.status(msg='Move release artifacts to public directory')
- private_dir = self.settings['download-server-private-dir']
- public_dir = self.settings['download-server-public-dir']
- self.create_directory_on_download_server(public_dir)
-
- # Move just the contents of the private dir, not the dir
- # itself (-mindepth). Avoid overwriting existing files (mv
- # -n).
- argv = ['find', private_dir, '-mindepth', '1',
- '-exec', 'mv', '-n', '{}', public_dir + '/.', ';']
-
- target = '{user}@{host}'.format(
- user=self.settings['download-server-username'],
- host=self.settings['download-server-address'])
- cliapp.ssh_runcmd(target, argv)
-
- def create_symlinks_to_new_release_artifacts(self, files):
- self.status(msg='FIXME: Create symlinks to new releas artifacts')
-
-
-def rsync_files_to_server(
- source_dir, source_filenames, user, host, target_dir):
-
- if not source_filenames:
- return
-
- argv = [
- 'rsync',
- '-a',
- '--progress',
- '--partial',
- '--human-readable',
- '--sparse',
- '--protect-args',
- '-0',
- '--files-from=-',
- source_dir,
- '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir),
- ]
-
- files_list = '\0'.join(filename for filename in source_filenames)
- cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None)
-
-
-def set_permissions_on_server(user, host, target_dir, filenames):
- # If we have no files, we can't form a valid command to run on the server
- if not filenames:
- return
- target = '{user}@{host}'.format(user=user, host=host)
- argv = ['xargs', '-0', 'chmod', '0644']
- files_list = ''.join(
- '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames)
- cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None)
-
-
-ReleaseUploader(description=__doc__).run()
diff --git a/scripts/release-upload.test.conf b/scripts/release-upload.test.conf
deleted file mode 100644
index 13227983..00000000
--- a/scripts/release-upload.test.conf
+++ /dev/null
@@ -1,10 +0,0 @@
-[config]
-download-server-address = localhost
-download-server-private-dir = /tmp/private
-download-server-public-dir = /tmp/public
-build-trove-host = ct-mcr-1.ducie.codethink.co.uk
-public-trove-host = localhost
-public-trove-username = root
-public-trove-artifact-dir = /tmp/artifacts
-release-artifact-dir = t.release-files
-morph-cmd = /home/root/git-morph
diff --git a/scripts/scriptslib.py b/scripts/scriptslib.py
deleted file mode 100644
index 53c6ca8e..00000000
--- a/scripts/scriptslib.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# Copyright (C) 2016 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Small library of useful things for the scripts that live here.
-
-import yaml
-import subprocess
-import os
-import sys
-
-aliases = {
- 'baserock:': 'git://%(trove)s/baserock/',
- 'freedesktop:': 'git://anongit.freedesktop.org/',
- 'github:': 'git://github.com/',
- 'gnome:': 'git://git.gnome.org/',
- 'upstream:': 'git://%(trove)s/delta/'
-}
-
-def parse_repo_alias(repo, trove_host='git.baserock.org'):
- global aliases
- remote = repo[:repo.find(':') + 1]
- aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()}
- try:
- return repo.replace(remote, aliases[remote])
- except KeyError as e:
- raise Exception("Unknown repo-alias \"%s\"" % repo)
-
-def definitions_root():
- return subprocess.check_output(
- ["git", "rev-parse", "--show-toplevel"]).strip()
-
-def load_yaml_file(yaml_file):
- with open(yaml_file, 'r') as f:
- return yaml.safe_load(f)
-
-
-class BaserockMeta(object):
- '''An object representing Baserock metadata contained in a Baserock
- system image, for available metadata formats'''
-
- def __init__(self):
- self.metas = {}
-
- def get_each(self):
- '''Yield an iterable for the whole list of metas'''
- for key in self.metas:
- yield self.metas[key]
-
- def get_name(self, name):
- '''Yield an iterable of metadata matched by name, e.g. `bash`'''
- for key in self.metas:
- if self.metas[key]['source-name'] == name:
- yield self.metas[key]
-
- def import_meta(self, meta_text):
- importers = (self.import_meta_ybd,
- self.import_meta_morph)
-
- for i in importers:
- try:
- i(meta_text)
- return
- except (KeyError, Exception) as err:
- pass
-
- # Shouldn't get here
- sys.stderr.write('Metadata format not recognised.\n'
- 'Error:\n')
- raise err
-
- def import_meta_morph(self, meta_text):
- self._add_meta(yaml.load(meta_text))
-
- def import_meta_ybd(self, meta_text):
- source = yaml.load(meta_text)
-
- null = '0' * 32
-
- if 'configuration' in source:
- # This is the deployment metadata, ignore
- return
- elif 'repo' not in source:
- kind = 'stratum'
- contents = 'components'
- source['repo'] = 'upstream:definitions'
- source['ref'] = null # No ref info
- else:
- kind = 'chunk'
- contents = 'components'
-
- repo = parse_repo_alias(source['repo'])
- source_name = '-'.join(
- source['products'][0]['artifact'].split('-')[:-1])
-
- # Needed until YBD provides cache-key in metadata
- if not 'cache-key' in source:
- source['cache-key'] = null
-
- for product in source['products']:
-
- self._add_meta({
- 'kind': kind,
- 'source-name': source_name,
- 'artifact-name': product['artifact'],
- 'contents': product[contents],
- 'repo': repo,
- 'repo-alias': source['repo'],
- 'sha1': source['ref'],
- 'original_ref': source['ref'],
- 'cache-key': source['cache-key']
- })
-
- def _add_meta(self, meta_dict):
- '''Validate and add a meta'''
-
- ignore = ('configuration',
- 'system-artifact-name')
-
- for i in ignore:
- if i in meta_dict:
- return
-
- required_fields = ('repo', 'sha1', 'contents')
- for f in required_fields:
- if not f in meta_dict:
- raise Exception('Metadata format not recognised, no '
- 'value for \'%s\'. Data: \'%s\''% (f, str(meta_dict)))
-
- self.metas[meta_dict['artifact-name']] = meta_dict
-
-
-def meta_load_from_dir(meta_dir_path):
- '''Read Baserock metadata from a directory'''
-
- files = [f for f in os.listdir(meta_dir_path)
- if os.path.isfile(os.path.join(meta_dir_path, f))]
-
- meta = BaserockMeta()
- for f in files:
- if f.endswith('.meta'):
- meta.import_meta(
- open(os.path.join(meta_dir_path, f), 'r').read())
-
- return meta
diff --git a/scripts/yaml-jsonschema b/scripts/yaml-jsonschema
deleted file mode 100755
index 64f52a79..00000000
--- a/scripts/yaml-jsonschema
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2015 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-'''A tool to validate YAML files against the JSON-Schema schemas.
-
-This wraps Python `jsonschema` module so that YAML schemas can be understood
-and YAML data can be validated.
-
-Usage: yaml-jsonschema SCHEMA INPUT1 [INPUT2, ...]
-
-'''
-
-
-import jsonschema
-import yaml
-
-import sys
-
-
-schema_file = sys.argv[1]
-input_files = sys.argv[2:]
-
-
-with open(schema_file) as f:
- schema = yaml.load(f)
-
-
-for input_file in input_files:
- with open(input_file) as f:
- data = yaml.load(f)
-
- try:
- jsonschema.validate(data, schema)
- print("%s: valid" % input_file)
- except jsonschema.ValidationError as e:
- # Print 'e' instead of 'e.message' for more information!
- print("%s: %s" % (input_file, e.message))