diff options
Diffstat (limited to 'old/scripts')
| -rwxr-xr-x | old/scripts/check-unpetrify-refs.py | 76 | ||||
| -rwxr-xr-x | old/scripts/cycle.sh | 61 | ||||
| -rw-r--r-- | old/scripts/licensecheck.pl | 604 | ||||
| -rwxr-xr-x | old/scripts/licensecheck.py | 201 | ||||
| -rwxr-xr-x | old/scripts/organize-morphologies.py | 255 | ||||
| -rwxr-xr-x | old/scripts/release-build | 192 | ||||
| -rw-r--r-- | old/scripts/release-build.test.conf | 6 | ||||
| -rwxr-xr-x | old/scripts/release-test | 400 | ||||
| -rwxr-xr-x | old/scripts/release-test-os | 526 | ||||
| -rwxr-xr-x | old/scripts/release-upload | 473 | ||||
| -rw-r--r-- | old/scripts/release-upload.test.conf | 10 | ||||
| -rw-r--r-- | old/scripts/scriptslib.py | 156 | ||||
| -rwxr-xr-x | old/scripts/yaml-jsonschema | 50 |
13 files changed, 3010 insertions, 0 deletions
diff --git a/old/scripts/check-unpetrify-refs.py b/old/scripts/check-unpetrify-refs.py new file mode 100755 index 00000000..c70b680d --- /dev/null +++ b/old/scripts/check-unpetrify-refs.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import glob +import argparse +import subprocess + +import scriptslib + + +''' +Script for checking unpetrify-refs in strata. + +Without args this script will check everything in strata/, or each stratum +given on the command-line and will print on stdout whether a chunk has +a missing or non-existent unpetrify-ref and if it fails to check the remote +(missing repo). +''' + +strata_dir = "strata" + +def ref_exists(remote, ref): + output = subprocess.check_output( + ["git", "ls-remote", remote, str(ref)], + stderr=subprocess.STDOUT).strip() + return True if output else False + +def main(): + parser = argparse.ArgumentParser( + description="Sanity checks unpetrify-refs in Baserock strata") + parser.add_argument("--trove-host", default="git.baserock.org", + help="Trove host to map repo aliases to") + parser.add_argument("strata", nargs="*", metavar="STRATA", + help="The strata to check (checks all by default)") + args = parser.parse_args() + + if args.strata: + strata = args.strata + else: + strata_path = os.path.join(scriptslib.definitions_root(), strata_dir) + strata = glob.glob("%s/*.morph" % strata_path) + + for stratum in strata: + path = os.path.relpath(stratum) + morphology = scriptslib.load_yaml_file(stratum) + for chunk in morphology['chunks']: + unpetrify_ref = chunk.get("unpetrify-ref") + if not unpetrify_ref: + print ("%s: '%s' has no unpetrify-ref!" % + (path, chunk['name'])) + continue + remote = scriptslib.parse_repo_alias(chunk['repo'], args.trove_host) + try: + if not ref_exists(remote, unpetrify_ref): + print ("%s: unpetrify-ref for '%s' is not present on the " + "remote (%s)!" % (path, chunk['name'], remote)) + except subprocess.CalledProcessError as e: + print ("%s: failed to ls-remote (%s) for chunk '%s':\n%s" % + (path, remote, chunk['name'], e.output.strip())) + +if __name__ == "__main__": + main() diff --git a/old/scripts/cycle.sh b/old/scripts/cycle.sh new file mode 100755 index 00000000..c0e2aa67 --- /dev/null +++ b/old/scripts/cycle.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +usage() { + echo "Usage: cycle.sh some-system some-cluster [newversion]" + echo + echo "This builds and deploys the current checked out version of" + echo "some-system, applying it as a self-upgrade to the system you" + echo "are working in, using configuration from some-cluster." + echo "The upgrade is labelled TEST by default, or [newversion] if" + echo "specified, and is set to be the default for next boot." +} + +if [ -z "$1" ] || [ -z "$2" ] || [ ! -z "$4" ] ; then + usage + exit 1 +fi + +newversion=TEST +if [ ! -z "$3" ] ; then + newversion=$3 + if (echo "$newversion" | grep ' ' > /dev/null 2>&1) ; then + echo 'Version label must not contain spaces.' + exit 1 + fi +fi + +if system-version-manager get-running | grep -q "^$newversion$"; then + echo "You are currently running the $newversion system." + echo "Maybe you want to boot into a different system version?" + exit 1 +fi + +set -e +set -v + +runningversion=`system-version-manager get-running` +system-version-manager set-default $runningversion +if system-version-manager list | grep -q "^$newversion$"; then + system-version-manager remove $newversion +fi + +morph gc +morph build "$1" + +sed -i "s|^- morph: .*$|- morph: $1|" "$2" +morph deploy --upgrade "$2" self.HOSTNAME=$(hostname) self.VERSION_LABEL=$newversion +system-version-manager list diff --git a/old/scripts/licensecheck.pl b/old/scripts/licensecheck.pl new file mode 100644 index 00000000..5b6d0d33 --- /dev/null +++ b/old/scripts/licensecheck.pl @@ -0,0 +1,604 @@ +#!/usr/bin/perl +# This script was originally based on the script of the same name from +# the KDE SDK (by dfaure@kde.org) +# +# This version is +# Copyright (C) 2007, 2008 Adam D. Barratt +# Copyright (C) 2012 Francesco Poli +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <https://www.gnu.org/licenses/>. + +=head1 NAME + +licensecheck - simple license checker for source files + +=head1 SYNOPSIS + +B<licensecheck> B<--help>|B<--version> + +B<licensecheck> [B<--no-conf>] [B<--verbose>] [B<--copyright>] +[B<-l>|B<--lines=>I<N>] [B<-i>|B<--ignore=>I<regex>] [B<-c>|B<--check=>I<regex>] +[B<-m>|B<--machine>] [B<-r>|B<--recursive>] +I<list of files and directories to check> + +=head1 DESCRIPTION + +B<licensecheck> attempts to determine the license that applies to each file +passed to it, by searching the start of the file for text belonging to +various licenses. + +If any of the arguments passed are directories, B<licensecheck> will add +the files contained within to the list of files to process. + +=head1 OPTIONS + +=over 4 + +=item B<--verbose>, B<--no-verbose> + +Specify whether to output the text being processed from each file before +the corresponding license information. + +Default is to be quiet. + +=item B<-l=>I<N>, B<--lines=>I<N> + +Specify the number of lines of each file's header which should be parsed +for license information. (Default is 60). + +=item B<-i=>I<regex>, B<--ignore=>I<regex> + +When processing the list of files and directories, the regular +expression specified by this option will be used to indicate those which +should not be considered (e.g. backup files, VCS metadata). + +=item B<-r>, B<--recursive> + +Specify that the contents of directories should be added +recursively. + +=item B<-c=>I<regex>, B<--check=>I<regex> + +Specify a pattern against which filenames will be matched in order to +decide which files to check the license of. + +The default includes common source files. + +=item B<--copyright> + +Also display copyright text found within the file + +=item B<-m>, B<--machine> + +Display the information in a machine readable way, i.e. in the form +<file><tab><license>[<tab><copyright>] so that it can be easily sorted +and/or filtered, e.g. with the B<awk> and B<sort> commands. +Note that using the B<--verbose> option will kill the readability. + +=item B<--no-conf>, B<--noconf> + +Do not read any configuration files. This can only be used as the first +option given on the command line. + +=back + +=head1 CONFIGURATION VARIABLES + +The two configuration files F</etc/devscripts.conf> and +F<~/.devscripts> are sourced by a shell in that order to set +configuration variables. Command line options can be used to override +configuration file settings. Environment variable settings are +ignored for this purpose. The currently recognised variables are: + +=over 4 + +=item B<LICENSECHECK_VERBOSE> + +If this is set to I<yes>, then it is the same as the B<--verbose> command +line parameter being used. The default is I<no>. + +=item B<LICENSECHECK_PARSELINES> + +If this is set to a positive number then the specified number of lines +at the start of each file will be read whilst attempting to determine +the license(s) in use. This is equivalent to the B<--lines> command line +option. + +=back + +=head1 LICENSE + +This code is copyright by Adam D. Barratt <I<adam@adam-barratt.org.uk>>, +all rights reserved; based on a script of the same name from the KDE +SDK, which is copyright by <I<dfaure@kde.org>>. +This program comes with ABSOLUTELY NO WARRANTY. +You are free to redistribute this code under the terms of the GNU +General Public License, version 2 or later. + +=head1 AUTHOR + +Adam D. Barratt <adam@adam-barratt.org.uk> + +=cut + +use strict; +use warnings; +use Getopt::Long qw(:config gnu_getopt); +use File::Basename; + +my $progname = basename($0); + +# From dpkg-source +my $default_ignore_regex = ' +# Ignore general backup files +(?:^|/).*~$| +# Ignore emacs recovery files +(?:^|/)\.#.*$| +# Ignore vi swap files +(?:^|/)\..*\.swp$| +# Ignore baz-style junk files or directories +(?:^|/),,.*(?:$|/.*$)| +# File-names that should be ignored (never directories) +(?:^|/)(?:DEADJOE|\.cvsignore|\.arch-inventory|\.bzrignore|\.gitignore)$| +# File or directory names that should be ignored +(?:^|/)(?:CVS|RCS|\.pc|\.deps|\{arch\}|\.arch-ids|\.svn|\.hg|_darcs|\.git| +\.shelf|_MTN|\.bzr(?:\.backup|tags)?)(?:$|/.*$) +'; + +# Take out comments and newlines +$default_ignore_regex =~ s/^#.*$//mg; +$default_ignore_regex =~ s/\n//sg; + +my $default_check_regex = '\.(c(c|pp|xx)?|h(h|pp|xx)?|f(77|90)?|go|p(l|m)|xs|sh|php|py(|x)|rb|java|js|vala|el|sc(i|e)|cs|pas|inc|dtd|xsl|mod|m|tex|mli?|(c|l)?hs)$'; + +my $modified_conf_msg; + +my %OPT=( + verbose => '', + lines => '', + noconf => '', + ignore => '', + check => '', + recursive => 0, + copyright => 0, + machine => 0, +); + +my $def_lines = 60; + +# Read configuration files and then command line +# This is boilerplate + +if (@ARGV and $ARGV[0] =~ /^--no-?conf$/) { + $modified_conf_msg = " (no configuration files read)"; + shift; +} else { + my @config_files = ('/etc/devscripts.conf', '~/.devscripts'); + my %config_vars = ( + 'LICENSECHECK_VERBOSE' => 'no', + 'LICENSECHECK_PARSELINES' => $def_lines, + ); + my %config_default = %config_vars; + + my $shell_cmd; + # Set defaults + foreach my $var (keys %config_vars) { + $shell_cmd .= qq[$var="$config_vars{$var}";\n]; + } + $shell_cmd .= 'for file in ' . join(" ", @config_files) . "; do\n"; + $shell_cmd .= '[ -f $file ] && . $file; done;' . "\n"; + # Read back values + foreach my $var (keys %config_vars) { $shell_cmd .= "echo \$$var;\n" } + my $shell_out = `/bin/bash -c '$shell_cmd'`; + @config_vars{keys %config_vars} = split /\n/, $shell_out, -1; + + # Check validity + $config_vars{'LICENSECHECK_VERBOSE'} =~ /^(yes|no)$/ + or $config_vars{'LICENSECHECK_VERBOSE'} = 'no'; + $config_vars{'LICENSECHECK_PARSELINES'} =~ /^[1-9][0-9]*$/ + or $config_vars{'LICENSECHECK_PARSELINES'} = $def_lines; + + foreach my $var (sort keys %config_vars) { + if ($config_vars{$var} ne $config_default{$var}) { + $modified_conf_msg .= " $var=$config_vars{$var}\n"; + } + } + $modified_conf_msg ||= " (none)\n"; + chomp $modified_conf_msg; + + $OPT{'verbose'} = $config_vars{'LICENSECHECK_VERBOSE'} eq 'yes' ? 1 : 0; + $OPT{'lines'} = $config_vars{'LICENSECHECK_PARSELINES'}; +} + +GetOptions(\%OPT, + "help|h", + "check|c=s", + "copyright", + "ignore|i=s", + "lines|l=i", + "machine|m", + "noconf|no-conf", + "recursive|r", + "verbose!", + "version|v", +) or die "Usage: $progname [options] filelist\nRun $progname --help for more details\n"; + +$OPT{'lines'} = $def_lines if $OPT{'lines'} !~ /^[1-9][0-9]*$/; +$OPT{'ignore'} = $default_ignore_regex if ! length $OPT{'ignore'}; +$OPT{'check'} = $default_check_regex if ! length $OPT{'check'}; + +if ($OPT{'noconf'}) { + fatal("--no-conf is only acceptable as the first command-line option!"); +} +if ($OPT{'help'}) { help(); exit 0; } +if ($OPT{'version'}) { version(); exit 0; } + +die "Usage: $progname [options] filelist\nRun $progname --help for more details\n" unless @ARGV; + +$OPT{'lines'} = $def_lines if not defined $OPT{'lines'}; + +my @files = (); +my @find_args = (); +my $files_count = @ARGV; + +push @find_args, qw(-maxdepth 1) unless $OPT{'recursive'}; +push @find_args, qw(-follow -type f -print); + +while (@ARGV) { + my $file = shift @ARGV; + + if (-d $file) { + open my $FIND, '-|', 'find', $file, @find_args + or die "$progname: couldn't exec find: $!\n"; + + while (<$FIND>) { + chomp; + next unless m%$OPT{'check'}%; + # Skip empty files + next if (-z $_); + push @files, $_ unless m%$OPT{'ignore'}%; + } + close $FIND; + } else { + next unless ($files_count == 1) or $file =~ m%$OPT{'check'}%; + push @files, $file unless $file =~ m%$OPT{'ignore'}%; + } +} + +while (@files) { + my $file = shift @files; + my $content = ''; + my $copyright_match; + my $copyright = ''; + my $license = ''; + my %copyrights; + + open (my $F, '<' ,$file) or die "Unable to access $file\n"; + while (<$F>) { + last if ($. > $OPT{'lines'}); + $content .= $_; + $copyright_match = parse_copyright($_); + if ($copyright_match) { + $copyrights{lc("$copyright_match")} = "$copyright_match"; + } + } + close($F); + + $copyright = join(" / ", reverse sort values %copyrights); + + print qq(----- $file header -----\n$content----- end header -----\n\n) + if $OPT{'verbose'}; + + $license = parselicense(clean_comments($content)); + + if ($OPT{'machine'}) { + print "$file\t$license"; + print "\t" . ($copyright or "*No copyright*") if $OPT{'copyright'}; + print "\n"; + } else { + print "$file: "; + print "*No copyright* " unless $copyright; + print $license . "\n"; + print " [Copyright: " . $copyright . "]\n" + if $copyright and $OPT{'copyright'}; + print "\n" if $OPT{'copyright'}; + } +} + +sub parse_copyright { + my $copyright = ''; + my $match; + + my $copyright_indicator_regex = ' + (?:copyright # The full word + |copr\. # Legally-valid abbreviation + |\x{00a9} # Unicode character COPYRIGHT SIGN + |\xc2\xa9 # Unicode copyright sign encoded in iso8859 + |\(c\) # Legally-null representation of sign + )'; + my $copyright_disindicator_regex = ' + \b(?:info(?:rmation)? # Discussing copyright information + |(notice|statement|claim|string)s? # Discussing the notice + |and|or|is|in|to # Part of a sentence + |(holder|owner)s? # Part of a sentence + |ownership # Part of a sentence + )\b'; + my $copyright_predisindicator_regex = '( + ^[#]define\s+.*\(c\) # #define foo(c) -- not copyright + )'; + + if ( ! m%$copyright_predisindicator_regex%ix) { + + if (m%$copyright_indicator_regex(?::\s*|\s+)(\S.*)$%ix) { + $match = $1; + + # Ignore lines matching "see foo for copyright information" etc. + if ($match !~ m%^\s*$copyright_disindicator_regex%ix) { + # De-cruft + $match =~ s/([,.])?\s*$//; + $match =~ s/$copyright_indicator_regex//igx; + $match =~ s/^\s+//; + $match =~ s/\s{2,}/ /g; + $match =~ s/\\@/@/g; + $copyright = $match; + } + } + } + + return $copyright; +} + +sub clean_comments { + local $_ = shift or return q{}; + + # Remove generic comments: look for 4 or more lines beginning with + # regular comment pattern and trim it. Fall back to old algorithm + # if no such pattern found. + my @matches = m/^\s*([^a-zA-Z0-9\s]{1,3})\s\w/mg; + if (@matches >= 4) { + my $comment_re = qr/\s*[\Q$matches[0]\E]{1,3}\s*/; + s/^$comment_re//mg; + } + + # Remove Fortran comments + s/^[cC] //gm; + tr/\t\r\n/ /; + + # Remove C / C++ comments + s#(\*/|/[/*])##g; + tr% A-Za-z.,@;0-9\(\)/-%%cd; + tr/ //s; + + return $_; +} + +sub help { + print <<"EOF"; +Usage: $progname [options] filename [filename ...] +Valid options are: + --help, -h Display this message + --version, -v Display version and copyright info + --no-conf, --noconf Don't read devscripts config files; must be + the first option given + --verbose Display the header of each file before its + license information + --lines, -l Specify how many lines of the file header + should be parsed for license information + (Default: $def_lines) + --check, -c Specify a pattern indicating which files should + be checked + (Default: '$default_check_regex') + --machine, -m Display in a machine readable way (good for awk) + --recursive, -r Add the contents of directories recursively + --copyright Also display the file's copyright + --ignore, -i Specify that files / directories matching the + regular expression should be ignored when + checking files + (Default: '$default_ignore_regex') + +Default settings modified by devscripts configuration files: +$modified_conf_msg +EOF +} + +sub version { + print <<"EOF"; +This is $progname, from the Debian devscripts package, version ###VERSION### +Copyright (C) 2007, 2008 by Adam D. Barratt <adam\@adam-barratt.org.uk>; based +on a script of the same name from the KDE SDK by <dfaure\@kde.org>. + +This program comes with ABSOLUTELY NO WARRANTY. +You are free to redistribute this code under the terms of the +GNU General Public License, version 2, or (at your option) any +later version. +EOF +} + +sub parselicense { + my ($licensetext) = @_; + + my $gplver = ""; + my $extrainfo = ""; + my $license = ""; + + if ($licensetext =~ /version ([^, ]+?)[.,]? (?:\(?only\)?.? )?(?:of the GNU (Affero )?(Lesser |Library )?General Public License )?(as )?published by the Free Software Foundation/i or + $licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License (?:as )?published by the Free Software Foundation[;,] version ([^, ]+?)[.,]? /i) { + + $gplver = " (v$1)"; + } elsif ($licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License, version (\d+(?:\.\d+)?)[ \.]/) { + $gplver = " (v$1)"; + } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) any later version/) { + $gplver = " (v$1 or later)"; + } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) version (\d(?:[\.-]\d+)*)/) { + $gplver = " (v$1 or v$2)"; + } + + if ($licensetext =~ /(?:675 Mass Ave|59 Temple Place|51 Franklin Steet|02139|02111-1307)/i) { + $extrainfo = " (with incorrect FSF address)$extrainfo"; + } + + if ($licensetext =~ /permission (?:is (also granted|given))? to link (the code of )?this program with (any edition of )?(Qt|the Qt library)/i) { + $extrainfo = " (with Qt exception)$extrainfo" + } + + if ($licensetext =~ /(All changes made in this file will be lost|DO NOT (EDIT|delete this file)|Generated (automatically|by|from)|generated.*file)/i) { + $license = "GENERATED FILE"; + } + + if ($licensetext =~ /((is free software.? )?you can redistribute (it|them) and\/or modify (it|them)|is licensed) under the terms of (version [^ ]+ of )?the (GNU (Library |Lesser )General Public License|LGPL)/i) { + $license = "LGPL$gplver$extrainfo $license"; + } + + if ($licensetext =~ /is free software.? you can redistribute (it|them) and\/or modify (it|them) under the terms of the (GNU Affero General Public License|AGPL)/i) { + $license = "AGPL$gplver$extrainfo $license"; + } + + if ($licensetext =~ /(is free software.? )?you (can|may) redistribute (it|them) and\/or modify (it|them) under the terms of (?:version [^ ]+ (?:\(?only\)? )?of )?the GNU General Public License/i) { + $license = "GPL$gplver$extrainfo $license"; + } + + if ($licensetext =~ /is distributed under the terms of the GNU General Public License,/ + and length $gplver) { + $license = "GPL$gplver$extrainfo $license"; + } + + if ($licensetext =~ /is distributed.*terms.*GPL/) { + $license = "GPL (unversioned/unknown version) $license"; + } + + if ($licensetext =~ /This file is part of the .*Qt GUI Toolkit. This file may be distributed under the terms of the Q Public License as defined/) { + $license = "QPL (part of Qt) $license"; + } elsif ($licensetext =~ /may (be distributed|redistribute it) under the terms of the Q Public License/) { + $license = "QPL $license"; + } + + if ($licensetext =~ /opensource\.org\/licenses\/mit-license\.php/) { + $license = "MIT/X11 (BSD like) $license"; + } elsif ($licensetext =~ /Permission is hereby granted, free of charge, to any person obtaining a copy of this software and(\/or)? associated documentation files \(the (Software|Materials)\), to deal in the (Software|Materials)/) { + $license = "MIT/X11 (BSD like) $license"; + } elsif ($licensetext =~ /Permission is hereby granted, without written agreement and without license or royalty fees, to use, copy, modify, and distribute this software and its documentation for any purpose/) { + $license = "MIT/X11 (BSD like) $license"; + } + + if ($licensetext =~ /Permission to use, copy, modify, and(\/or)? distribute this software for any purpose with or without fee is hereby granted, provided.*copyright notice.*permission notice.*all copies/) { + $license = "ISC $license"; + } + + if ($licensetext =~ /THIS SOFTWARE IS PROVIDED .*AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY/) { + if ($licensetext =~ /All advertising materials mentioning features or use of this software must display the following acknowledge?ment.*This product includes software developed by/i) { + $license = "BSD (4 clause) $license"; + } elsif ($licensetext =~ /(The name(?:\(s\))? .*? may not|Neither the (names? .*?|authors?) nor the names of( (its|their|other|any))? contributors may) be used to endorse or promote products derived from this software/i) { + $license = "BSD (3 clause) $license"; + } elsif ($licensetext =~ /Redistributions of source code must retain the above copyright notice/i) { + $license = "BSD (2 clause) $license"; + } else { + $license = "BSD $license"; + } + } + + if ($licensetext =~ /Mozilla Public License,? (Version|v\.) (\d+(?:\.\d+)?)/) { + $license = "MPL (v$2) $license"; + } + + if ($licensetext =~ /Released under the terms of the Artistic License ([^ ]+)/) { + $license = "Artistic (v$1) $license"; + } + + if ($licensetext =~ /is free software under the Artistic [Ll]icense/) { + $license = "Artistic $license"; + } + + if ($licensetext =~ /This program is free software; you can redistribute it and\/or modify it under the same terms as Perl itself/) { + $license = "Perl $license"; + } + + if ($licensetext =~ /under the Apache License, Version ([^ ]+)/) { + $license = "Apache (v$1) $license"; + } + + if ($licensetext =~ /(THE BEER-WARE LICENSE)/i) { + $license = "Beerware $license"; + } + + if ($licensetext =~ /This source file is subject to version ([^ ]+) of the PHP license/) { + $license = "PHP (v$1) $license"; + } + + if ($licensetext =~ /under the terms of the CeCILL /) { + $license = "CeCILL $license"; + } + + if ($licensetext =~ /under the terms of the CeCILL-([^ ]+) /) { + $license = "CeCILL-$1 $license"; + } + + if ($licensetext =~ /under the SGI Free Software License B/) { + $license = "SGI Free Software License B $license"; + } + + if ($licensetext =~ /is in the public domain/i) { + $license = "Public domain $license"; + } + + if ($licensetext =~ /terms of the Common Development and Distribution License(, Version ([^(]+))? \(the License\)/) { + $license = "CDDL " . ($1 ? "(v$2) " : '') . $license; + } + + if ($licensetext =~ /Microsoft Permissive License \(Ms-PL\)/) { + $license = "Ms-PL $license"; + } + + if ($licensetext =~ /Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by this license \(the \"Software\"\)/ or + $licensetext =~ /Boost Software License([ ,-]+Version ([^ ]+)?(\.))/i) { + $license = "BSL " . ($1 ? "(v$2) " : '') . $license; + } + + if ($licensetext =~ /PYTHON SOFTWARE FOUNDATION LICENSE (VERSION ([^ ]+))/i) { + $license = "PSF " . ($1 ? "(v$2) " : '') . $license; + } + + if ($licensetext =~ /The origin of this software must not be misrepresented.*Altered source versions must be plainly marked as such.*This notice may not be removed or altered from any source distribution/ or + $licensetext =~ /see copyright notice in zlib\.h/) { + $license = "zlib/libpng $license"; + } elsif ($licensetext =~ /This code is released under the libpng license/) { + $license = "libpng $license"; + } + + if ($licensetext =~ /Do What The Fuck You Want To Public License, Version ([^, ]+)/i) { + $license = "WTFPL (v$1) $license"; + } + + if ($licensetext =~ /Do what The Fuck You Want To Public License/i) { + $license = "WTFPL $license"; + } + + if ($licensetext =~ /(License WTFPL|Under (the|a) WTFPL)/i) { + $license = "WTFPL $license"; + } + + $license = "UNKNOWN" if (!length($license)); + + # Remove trailing spaces. + $license =~ s/\s+$//; + + return $license; +} + +sub fatal { + my ($pack,$file,$line); + ($pack,$file,$line) = caller(); + (my $msg = "$progname: fatal error at line $line:\n@_\n") =~ tr/\0//d; + $msg =~ s/\n\n$/\n/; + die $msg; +} diff --git a/old/scripts/licensecheck.py b/old/scripts/licensecheck.py new file mode 100755 index 00000000..08d0e1b4 --- /dev/null +++ b/old/scripts/licensecheck.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import argparse +import errno +import os +import pipes +import re +import string +import subprocess +import sys +import tempfile + +import scriptslib + + +gpl3_chunks = ("autoconf", + "autoconf-tarball", + "automake", + "bash", + "binutils", + "bison", + "ccache", + "cmake", + "flex", + "gawk", + "gcc", + "gdbm", + "gettext-tarball", + "gperf", + "groff", + "libtool", + "libtool-tarball", + "m4-tarball", + "make", + "nano", + "patch", + "rsync", + "texinfo-tarball") + + +def license_file_name(repo_name, sha, licenses_dir): + license_file = os.path.join(licenses_dir, repo_name + '-' + sha) + return license_file + + +def check_license(repo_name, sha, clone_path, license_file): + + licenses_dir = os.path.dirname(license_file) + + try: + os.makedirs(licenses_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + _, license_file_temp = tempfile.mkstemp(dir=licenses_dir) + with open(license_file_temp,"wb") as out: + sys.stderr.write("Checking license of '%s' ...\n" % repo_name) + with open(os.devnull, 'w') as devnull: + subprocess.check_call("perl scripts/licensecheck.pl -r " + + pipes.quote(clone_path) + "|cut -d: -f2- | sort -u", + stdout=out, stderr=devnull, shell=True) + + os.rename(license_file_temp, license_file) + return license_file + + +def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir): + repo_name = re.split('/|:',repo)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] + + repo_url = scriptslib.parse_repo_alias(repo) + + # Check if ref is sha1 to speedup + if len(ref) == 40 and all(c in string.hexdigits for c in ref): + license_file = license_file_name(repo_name, ref, licenses_dir) + if os.path.isfile(license_file): + return (repo, license_file) + + clone_path = os.path.join(repos_dir, repo_name) + + if os.path.isdir(clone_path): + sys.stderr.write("Updating repo '%s' ...\n" % repo_name) + with open(os.devnull, 'w') as devnull: + subprocess.check_call([ + "git", "remote", "update", "origin", "--prune"], + stderr=devnull, stdout=devnull, cwd=clone_path) + # Update submodules + subprocess.check_call( + ["git", "submodule", "update", "--recursive"], + stderr=devnull, stdout=devnull, cwd=clone_path) + subprocess.check_call(["git", "checkout", ref], stderr=devnull, + stdout=devnull, cwd=clone_path) + else: + sys.stderr.write("Getting repo '%s' ...\n" % repo_name) + with open(os.devnull, 'w') as devnull: + try: + # Attempt to use morph to obtain a repository, from morph's + # existing local git cache if possible + subprocess.check_call( + ["morph", "get-repo", name, clone_path], + stdout=devnull, stderr=devnull) + + except (OSError, subprocess.CalledProcessError): + # Fall back to git clone, when morph hasn't been found on the + # system, or otherwise fails to get a repo. This is required + # where morph isn't available, e.g. when using YBD to build. + # YBD currently doesn't offer a similar 'get-repo' feature. + sys.stderr.write("Falling back to git clone.\n") + subprocess.check_call( + ["git", "clone", "--recursive", repo_url, clone_path], + stdout=devnull, stderr=devnull) # also clone submodules + + sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=clone_path).strip() + + license_file = license_file_name(repo_name, sha, licenses_dir) + if os.path.isfile(license_file): + return (repo, license_file) + + return (repo, check_license(repo_name, sha, clone_path, license_file)) + + +def check_stratum(stratum_file, repos_dir, licenses_dir): + stratum = scriptslib.load_yaml_file(stratum_file) + license_files = [] + for chunk in stratum['chunks']: + + name = chunk["name"] + build_mode = chunk.get("build-mode") # Allowed to be None + + # Don't include bootstrap chunks and stripped gplv3 chunks + if name in gpl3_chunks or build_mode == "bootstrap": + continue + repo = chunk["repo"] + ref = chunk["ref"] + yield check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir) + + +def main(): + + parser = argparse.ArgumentParser( + description='Checks licenses of the components of a given System.') + parser.add_argument('system', metavar='SYSTEM', type=str, + help='System to check for licenses') + parser.add_argument('--repos-dir', default="./repos", + help='DIR to clone all the repos (default ./repos)') + parser.add_argument('--licenses-dir', default="./licenses", + help='DIR to store chunk license files (default ./licenses)') + + args = parser.parse_args() + + if not os.path.exists(args.repos_dir): + os.makedirs(args.repos_dir) + + system = scriptslib.load_yaml_file(args.system) + license_files = [] + for stratum in system['strata']: + stratum_file = stratum['morph'] + stratum_path = os.path.join(scriptslib.definitions_root(), stratum_file) + license_files.extend(check_stratum(stratum_path, args.repos_dir, args.licenses_dir)) + + for chunk_repo, chunk_license in license_files: + try: + # Print repo name + sys.stdout.write("%s\n%s\n" % (chunk_repo, '-' * len(chunk_repo))) + + # Print license file of the repo + with open(chunk_license, 'r') as f: + for line in f: + sys.stdout.write(line) + sys.stdout.write("\n") + except IOError: + # stdout is closed, no point in continuing + # Attempt to close them explicitly to prevent cleanup problems: + try: + sys.stdout.flush() + sys.stdout.close() + except IOError: + pass + finally: + exit() + + +if __name__ == "__main__": + main() diff --git a/old/scripts/organize-morphologies.py b/old/scripts/organize-morphologies.py new file mode 100755 index 00000000..3072c8f8 --- /dev/null +++ b/old/scripts/organize-morphologies.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python +# Copyright (C) 2014-2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import json +import morphlib +import os +import subprocess +import sys +import urllib +import urllib2 +import urlparse +import yaml +import re +import errno + +''' organize-morphologies.py: +Tool for organizing morphologies in definitions. + +This script will move: + - cluster morphologies into clusters directory + - system morphologies into systems directory + - stratum morphologies into strata directory + +This script will download the chunk morphologies for every stratum +and placed into strata/stratum_which_the_chunk_belongs_to directory. + +It also modifies the morphologies fields which points to some morpholgy +which has been moved. +''' + + +def make_request(path): + server_url = 'http://git.baserock.org:8080/' + url = urlparse.urljoin(server_url, '/1.0/%s' % path) + handle = urllib2.urlopen(url) + return handle.read() + +def quote(*args): + return tuple(urllib.quote(string) for string in args) + +def cat_file(repo, ref, filename): + return make_request('files?repo=%s&ref=%s&filename=%s' % + quote(repo, ref, filename)) + +# NOTE: This function reimplement part of morphlib's loader +def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'): + '''This function receives the name or the morph field of one morphology + and returns the path of the morphology depending on the name, kind and + if it belongs to other morphologies. + ''' + # Dictionary which match morphology's kind and morphology's + # directory in definitions.git + morph_dir = { 'chunk': 'chunks', 'stratum': 'strata', + 'system':'systems', 'cluster': 'clusters'} + # For chunks morphologies we need to know to which stratums + # belongs this chunk. + if morph_kind == 'chunk': + if belongs_to == '': + raise morphlib.Error('Chunk morphologies need the stratum name' + 'to create the path. Please add the stratum' + 'which belongs this morphology') + # Get the name of the chunk which we assume is at the end + # of the morph file + if '/' in morph_field: + morph_field = os.path.basename(morph_field) + + # Add the stratum name to the chunk name + morph_field = os.path.join(belongs_to, morph_field) + + # Reset the kind to stratum because chunk contains stratum + # name in its path. + morph_kind = 'stratum' + + # Add the morphology path to the morph field. + if not morph_field.startswith(morph_dir[morph_kind]): + morph_field = os.path.join(morph_dir[morph_kind], morph_field) + + # Add the morphology suffix if the morphology. + if not morph_field.endswith('.morph'): + morph_field = morph_field + '.morph' + + return morph_field + +def create_directory(name, path): + directory = os.path.join(path, name) + try: + os.makedirs(directory) + except OSError as err: + if err.errno != errno.EEXIST: + raise err + else: + pass + return directory + +def move_file(morph, directory, path, loader): + if not morph.filename.startswith(directory): + filename = os.path.basename(morph.filename) + new_location = os.path.join(path, filename) + print '\nMoving %s into %s' % (filename, new_location) + subprocess.call(['git', 'mv', morph.filename, new_location]) + morph.filename = new_location + loader.unset_defaults(morph) + loader.save_to_file(morph.filename, morph) + +def load_and_fix_chunk(chunk_str, loader, name): + try: + chunk_morph = loader.load_from_string(chunk_str) + except morphlib.morphloader.InvalidFieldError as err: + if "comments" in str(err): + # This error is caused because there are old morphologies which + # contain the field "comments" instead of "description". + # Replacing "comments" field by "description" will allow the morphology + # to pass parse_morphology_text check and ready to be written to a file. + fixed_chunk = loader.parse_morphology_text(chunk_str, name) + fixed_chunk['description'] = fixed_chunk.pop('comments') + print "WARNING: Invalid 'comments' field in " \ + "%s corrected to 'description'" % name + chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name) + elif "buildsystem" in str(err): + # This error is caused because a typo in a morphology which + # has a field "buildsystem" instead of "build-system". + fixed_chunk = loader.parse_morphology_text(chunk_str, name) + fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem') + print "WARNING: Invalid 'buildsystem' field in %s" \ + "corrected to 'build-system'" % name + chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name) + else: + print "ERROR: %s in chunk %s" %(err, name) + raise err + except morphlib.morphloader.MorphologyNotYamlError as err: + print "WARNING: %s in chunk %s is not valid YAML, " \ + "attempting to fix..." %(err, name) + # This error is caused because there are old morphologies written + # in JSON which contain '\t' characters. When try to load this + # kind of morphologies load_from_string fails when parse_morphology_text. + # Removing this characters will make load_from_string to load the morphology + # and translate it into a correct yaml format. + fixed_chunk = chunk_str.replace('\t','') + print "INFO: %s successfully fixed" % name + chunk_morph = load_and_fix_chunk(fixed_chunk, loader, name) + return chunk_morph + +def move_clusters(morphs, path, loader): + kind = 'system' + directory = 'clusters' + # Move cluster morphologies to clusters folder fixing their dependent + # morphologies which are systems. + full_path = create_directory(directory, path) + for morph in morphs: + all_systems = morph['systems'][:] + for system in morph['systems']: + all_systems.extend(system.get('subsystems', [])) + # Add the correct path to the morph fields for systems and subsystems + for field in all_systems: + field['morph'] = sanitise_morphology_path(field['morph'], kind) + move_file(morph, directory, full_path, loader) + +def move_systems(morphs, path, loader): + kind = 'stratum' + directory = 'systems' + # Move system morphologies to systems folder fixing their dependent + # morphologies which are strata. + full_path = create_directory(directory, path) + for morph in morphs: + # Add name field and the correct path to the stratum on the morph + # fields in strata. + for field in morph['strata']: + field['name'] = os.path.basename(field['morph']) + field['morph'] = sanitise_morphology_path(field['morph'], kind) + move_file(morph, directory, full_path, loader) + +def download_chunks(morph, loader): + # Download chunks morphologies defined on the stratum and + # add them to the directory tree. + for chunk in morph['chunks']: + name = chunk['name'] + '.morph' + try: + chunk['morph'] = sanitise_morphology_path(chunk['morph'], 'chunk', morph['name']) + except KeyError as err: + if 'morph' in str(err): + chunk['morph'] = sanitise_morphology_path(chunk['name'], 'chunk', morph['name']) + else: + raise err + ref = chunk['ref'] + repo = scriptslib.parse_repo_alias(chunk['repo']) + try: + print "\nDownloading %s from %s into %s" %(name, repo, chunk['morph']) + chunk_str = cat_file(repo, ref, name) + except urllib2.HTTPError as err: + # If there is no morphology in the repository we assume that the morphology + # system will be autodetected, so we don't have to create a new one + # unless we shut down the autodetecting system (fallback system). + if err.code == 404: + print 'INFO: Morph will fall-back to build-time' \ + 'autodetection for %s' %(name) + # Remove morph field from autodetected chunks + del chunk['morph'] + else: + loaded_chunk = load_and_fix_chunk(chunk_str, loader, name) + loader.unset_defaults(loaded_chunk) + loader.save_to_file(chunk['morph'], loaded_chunk) + +def move_strata(morphs, path, loader): + # Create strata directory + strata_dir = 'strata/' + strata_path = create_directory(strata_dir, path) + for morph in morphs: + # Create stratum directory where downloading its chunks. + stratum_path = strata_path + morph['name'] + stratum_dir = create_directory(stratum_path, path) + + # Download chunks which belongs to the stratum + download_chunks(morph, loader) + + # Add to build-depends the correct path to the dependent stratum morphologies. + for build_depends in morph['build-depends']: + build_depends['morph'] = sanitise_morphology_path(build_depends['morph'], 'stratum') + # Move stratum morphologies to strata + move_file(morph, strata_dir, strata_path, loader) + +def main(): + # Load all morphologies in the definitions repo + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + morphs = [m for m in sb.load_all_morphologies(loader)] + + # Clasify the morphologies regarding of their kind field + morphologies = { kind: [m for m in morphs if m['kind'] == kind] + for kind in ('chunk', 'stratum', 'system', 'cluster') } + + for kind, morphs in morphologies.iteritems(): + print 'There are: %d %s.\n' %(len(morphs), kind) + + # Get the path from definitions repo + definitions_repo = sb.get_git_directory_name(sb.root_repository_url) + + # Move the morphologies to its directories + move_clusters(morphologies['cluster'], definitions_repo, loader) + move_systems(morphologies['system'], definitions_repo, loader) + move_strata(morphologies['stratum'], definitions_repo, loader) + +main() diff --git a/old/scripts/release-build b/old/scripts/release-build new file mode 100755 index 00000000..cb62f661 --- /dev/null +++ b/old/scripts/release-build @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import morphlib +import os +import subprocess +import sys +import time + + +class Build(object): + '''A single distbuild instance.''' + + def __init__(self, name, arch, app): + self.system_name = name + + controller_netloc = app.controllers[arch].split(':') + controller_args = [ + '--controller-initiator-address=%s' % controller_netloc[0], + ] + if len(controller_netloc) > 1: + controller_args.append( + '--controller-initiator-port=%s' % controller_netloc[1]) + + self.command = ['morph', 'distbuild', '--local-changes=ignore'] + self.command += controller_args + [self.system_name] + + def start(self): + self.process = subprocess.Popen(self.command) + + def completed(self): + return (self.process.poll() is not None) + + +class ReleaseApp(cliapp.Application): + + '''Cliapp app that handles distbuilding and deploying a cluster.''' + + def add_settings(self): + self.settings.string_list(['controllers'], + 'a list of distbuild controllers and their ' + 'architecture') + + self.settings.string(['trove-host'], + 'hostname of Trove instance') + + self.settings.string(['artifact-cache-server'], + 'server to fetch artifacts from', default=None) + + self.settings.string(['release-number'], + 'Baserock version of the systems being built', + default='yy.ww') + + def error(self, message): + raise cliapp.AppException(message) + + def check_args(self, args): + if len(args) == 0: + self.error( + "Please pass the name of the release cluster (e.g. " + "clusters/release.morph)") + + if len(args) > 1: + self.error("Too many arguments given.") + + def process_args(self, args): + '''Process the command line''' + self.controllers = {} + controllers_list = self.settings['controllers'] + + for item in controllers_list: + arch, controller = item.split(':', 1) + self.controllers[arch] = controller + + defs_repo = morphlib.definitions_repo.open( + '.', search_for_root=True) + self.loader = defs_repo.get_morphology_loader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) + + self.check_args(args) + + cluster_name = args[0] + cluster, cluster_path = self.load_morphology(cluster_name) + + builds = self.prepare_builds(cluster) + for build in builds: + build.start() + + while not all(build.completed() for build in builds): + time.sleep(1) + + fail = False + for build in builds: + if build.process.returncode != 0: + fail = True + sys.stderr.write( + 'Building failed for %s\n' % build.system_name) + if fail: + raise cliapp.AppException('Building of systems failed') + + if not os.path.exists('release'): + os.mkdir('release') + self.deploy_images(cluster, cluster_path) + + def load_morphology(self, name, kind=None): + path = morphlib.util.sanitise_morphology_path(name) + morph = self.loader.load_from_string( + self.finder.read_file(path)) + if kind: + assert morph['kind'] == kind + return morph, path + + def iterate_systems(self, system_list): + for system in system_list: + yield system['morph'] + if 'subsystems' in system: + for subsystem in self.iterate_systems(system['subsystems']): + yield subsystem + + def prepare_builds(self, cluster): + '''Prepare a list of builds''' + systems = set(self.iterate_systems(cluster['systems'])) + builds = [] + for system_name in systems: + system, _ = self.load_morphology(system_name) + if system['arch'] in self.controllers: + builds.append(Build(system_name, system['arch'], self)) + else: + print("Unable to build %s: no %s distbuild available" % + (system_name, system['arch'])) + return builds + + def deploy_images(self, cluster, cluster_path): + version_label = 'baserock-%s' % self.settings['release-number'] + outputs = {} + + for system in cluster['systems']: + morphology_name = system['morph'] + morphology = self.load_morphology(morphology_name)[0] + if morphology['arch'] not in self.controllers: + continue + + for deployment_name, deployment_info in system['deploy'].iteritems(): + # The release.morph cluster must specify a basename for the file, + # of name and extension. This script knows about name, but it + # can't find out the appropriate file extension without second + # guessing the behaviour of write extensions. + basename = deployment_info['location'] + + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In %s: system %s.location should be just the base name, ' + 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name)) + + filename = os.path.join('release', '%s-%s' % (version_label, basename)) + if os.path.exists(filename): + self.output.write('Reusing existing deployment of %s\n' % filename) + else: + self.output.write('Creating %s from release.morph\n' % filename) + self.deploy_single_image(cluster_path, deployment_name, filename, version_label) + + def deploy_single_image(self, cluster_path, name, location, version_label): + deploy_command = [ + 'morph', 'deploy', cluster_path, name, + '--trove-host=%s' % self.settings['trove-host']] + artifact_server = self.settings['artifact-cache-server'] + if artifact_server is not None: + deploy_command.append('--artifact-cache-server=%s' % artifact_server) + deploy_command.extend(( + '%s.location=%s' % (name, location), + '%s.VERSION_LABEL=%s' % (name, version_label) + )) + + cliapp.runcmd(deploy_command, stdout=sys.stdout) + + +ReleaseApp().run() diff --git a/old/scripts/release-build.test.conf b/old/scripts/release-build.test.conf new file mode 100644 index 00000000..50083352 --- /dev/null +++ b/old/scripts/release-build.test.conf @@ -0,0 +1,6 @@ +[config] +trove-host = ct-mcr-1.ducie.codethink.co.uk +controllers = x86_64:ct-mcr-1-distbuild-x86-64-majikthise-controller.dyn.ducie.codethink.co.uk, + x86_32:ct-mcr-1-distbuild-x86-32-majikthise-controller.dyn.ducie.codethink.co.uk, + armv7lhf:ct-mcr-1-distbuild-armv7lhf-jetson.dyn.ducie.codethink.co.uk +release-number = 14.29 diff --git a/old/scripts/release-test b/old/scripts/release-test new file mode 100755 index 00000000..4dcc6f76 --- /dev/null +++ b/old/scripts/release-test @@ -0,0 +1,400 @@ +#!/usr/bin/env python +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''release-test + +This script deploys the set of systems in the cluster morphology it is +instructed to read, to test that they work correctly. + +''' + +import cliapp +import os +import pipes +import shlex +import shutil +import socket +import tempfile +import time +import uuid + +import morphlib + + +class MorphologyHelper(object): + + def __init__(self): + self.defs_repo = morphlib.definitions_repo.open( + '.', search_for_root=True) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) + + def load_morphology(self, path): + text = self.finder.read_file(path) + return self.loader.load_from_string(text) + + @classmethod + def iterate_systems(cls, systems_list): + for system in systems_list: + yield morphlib.util.sanitise_morphology_path(system['morph']) + if 'subsystems' in system: + for subsystem in cls.iterate_systems(system['subsystems']): + yield subsystem + + def iterate_cluster_deployments(cls, cluster_morph): + for system in cluster_morph['systems']: + path = morphlib.util.sanitise_morphology_path(system['morph']) + defaults = system.get('deploy-defaults', {}) + for name, options in system['deploy'].iteritems(): + config = dict(defaults) + config.update(options) + yield path, name, config + + def load_cluster_systems(self, cluster_morph): + for system_path in set(self.iterate_systems(cluster_morph['systems'])): + system_morph = self.load_morphology(system_path) + yield system_path, system_morph + + +class TimeoutError(cliapp.AppException): + + """Error to be raised when a connection waits too long""" + + def __init__(self, msg): + super(TimeoutError, self).__init__(msg) + + +class VMHost(object): + + def __init__(self, user, address, disk_path): + self.user = user + self.address = address + self.disk_path = disk_path + + @property + def ssh_host(self): + return '{user}@{address}'.format(user=self.user, address=self.address) + + def runcmd(self, *args, **kwargs): + cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) + + def virsh(self, *args, **kwargs): + self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs) + + +class DeployedSystemInstance(object): + + def __init__(self, deployment, config, host_machine, vm_id, rootfs_path): + self.deployment = deployment + self.config = config + # TODO: Stop assuming test machine can DHCP and be assigned its + # hostname in the deployer's resolve search path. + self.ip_address = self.config['HOSTNAME'] + self.host_machine = host_machine + self.vm_id = vm_id + self.rootfs_path = rootfs_path + + @property + def ssh_host(self): + # TODO: Stop assuming we ssh into test instances as root + return 'root@{host}'.format(host=self.ip_address) + + def runcmd(self, argv, chdir='.', **kwargs): + ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] + cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] + cmd += argv + ssh_cmd.append(' '.join(map(pipes.quote, cmd))) + return cliapp.runcmd(ssh_cmd, **kwargs) + + def _wait_for_dhcp(self, timeout): + '''Block until given hostname resolves successfully. + + Raises TimeoutError if the hostname has not appeared in 'timeout' + seconds. + + ''' + start_time = time.time() + while True: + try: + socket.gethostbyname(self.ip_address) + return + except socket.gaierror: + pass + if time.time() > start_time + timeout: + raise TimeoutError("Host %s did not appear after %i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_ssh(self, timeout): + """Wait until the deployed VM is responding via SSH""" + start_time = time.time() + while True: + try: + self.runcmd(['true'], stdin=None, stdout=None, stderr=None) + return + except cliapp.AppException: + # TODO: Stop assuming the ssh part of the command is what failed + if time.time() > start_time + timeout: + raise TimeoutError("%s sshd did not start after %i seconds" + % (self.ip_address, timeout)) + time.sleep(0.5) + + def wait_until_online(self, timeout=10): + self._wait_for_dhcp(timeout) + self._wait_for_ssh(timeout) + + def delete(self): + # Stop and remove VM + try: + self.host_machine.virsh('destroy', self.vm_id) + except cliapp.AppException as e: + # TODO: Stop assuming that destroy failed because it wasn't running + pass + try: + self.host_machine.virsh('undefine', self.vm_id, '--remove-all-storage') + except cliapp.AppException as e: + # TODO: Stop assuming that undefine failed because it was + # already removed + pass + + +class Deployment(object): + + def __init__(self, cluster_path, name, deployment_config, host_machine): + self.cluster_path = cluster_path + self.name = name + self.deployment_config = deployment_config + self.host_machine = host_machine + + @staticmethod + def _ssh_host_key_exists(hostname): + """Check if an ssh host key exists in known_hosts""" + if not os.path.exists('/root/.ssh/known_hosts'): + return False + with open('/root/.ssh/known_hosts', 'r') as known_hosts: + return any(line.startswith(hostname) for line in known_hosts) + + def _update_known_hosts(self): + if not self._ssh_host_key_exists(self.host_machine.address): + with open('/root/.ssh/known_hosts', 'a') as known_hosts: + cliapp.runcmd(['ssh-keyscan', self.host_machine.address], + stdout=known_hosts) + + @staticmethod + def _generate_sshkey_config(tempdir, config): + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write('0040700 0 0 /root/.ssh\n') + f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') + authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') + os.makedirs(os.path.dirname(authkeys)) + with open(authkeys, 'w') as auth_f: + with open('/root/.ssh/id_rsa.pub', 'r') as key_f: + shutil.copyfileobj(key_f, auth_f) + + install_files = shlex.split(config.get('INSTALL_FILES', '')) + install_files.append(manifest) + yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) + + def deploy(self): + self._update_known_hosts() + + hostname = str(uuid.uuid4()) + vm_id = hostname + image_base = self.host_machine.disk_path + rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, + hostname=hostname) + loc = 'kvm+ssh://{ssh_host}/{id}/{path}'.format( + ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) + + options = { + 'type': 'kvm', + 'location': loc, + 'AUTOSTART': 'True', + 'HOSTNAME': hostname, + 'DISK_SIZE': '20G', + 'RAM_SIZE': '2G', + 'VERSION_LABEL': 'release-test', + } + + tempdir = tempfile.mkdtemp() + try: + options.update( + self._generate_sshkey_config(tempdir, + self.deployment_config)) + + args = ['morph', 'deploy', self.cluster_path, self.name] + for k, v in options.iteritems(): + args.append('%s.%s=%s' % (self.name, k, v)) + cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) + + config = dict(self.deployment_config) + config.update(options) + + return DeployedSystemInstance(self, config, self.host_machine, + vm_id, rootpath) + finally: + shutil.rmtree(tempdir) + + +class ReleaseApp(cliapp.Application): + + """Cliapp application which handles automatic builds and tests""" + + def add_settings(self): + """Add the command line options needed""" + group_main = 'Program Options' + self.settings.string_list(['deployment-host'], + 'ARCH:HOST:PATH that VMs can be deployed to', + default=None, + group=group_main) + self.settings.string(['trove-host'], + 'Address of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['trove-id'], + 'ID of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['build-ref-prefix'], + 'Prefix of build branches for test systems', + default=None, + group=group_main) + + @staticmethod + def _run_tests(instance, system_path, system_morph, + (trove_host, trove_id, build_ref_prefix), + morph_helper, systems): + instance.wait_until_online() + + tests = [] + def baserock_build_test(instance): + instance.runcmd(['git', 'config', '--global', 'user.name', + 'Test Instance of %s' % instance.deployment.name]) + instance.runcmd(['git', 'config', '--global', 'user.email', + 'ci-test@%s' % instance.config['HOSTNAME']]) + instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', + '/src/tmp']) + def morph_cmd(*args, **kwargs): + # TODO: decide whether to use cached artifacts or not by + # adding --artifact-cache-server= --cache-server= + argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', + '--tempdir=/src/tmp', '--log-max=100M', + '--trove-host', trove_host, '--trove-id', trove_id, + '--build-ref-prefix', build_ref_prefix] + argv.extend(args) + instance.runcmd(argv, **kwargs) + + repo = morph_helper.sb.root_repository_url + ref = morph_helper.defs_repo.HEAD + sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref) + morph_cmd('init', '/src/ws') + chdir = '/src/ws' + + morph_cmd('checkout', repo, ref, chdir=chdir) + # TODO: Add a morph subcommand that gives the path to the root repository. + repo_path = os.path.relpath( + morph_helper.sb.get_git_directory_name(repo), + morph_helper.sb.root_directory) + chdir = os.path.join(chdir, ref, repo_path) + + instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) + print 'Building test systems for {sys}'.format(sys=system_path) + for to_build_path, to_build_morph in systems.iteritems(): + if to_build_morph['arch'] == system_morph['arch']: + print 'Test building {path}'.format(path=to_build_path) + morph_cmd('build', to_build_path, chdir=chdir, + stdin=None, stdout=None, stderr=None) + print 'Finished Building test systems' + + def python_smoke_test(instance): + instance.runcmd(['python', '-c', 'print "Hello World"']) + + # TODO: Come up with a better way of determining which tests to run + if 'devel' in system_path: + tests.append(baserock_build_test) + else: + tests.append(python_smoke_test) + + for test in tests: + test(instance) + + def deploy_and_test_systems(self, cluster_path, + deployment_hosts, build_test_config): + """Run the deployments and tests""" + + version = 'release-test' + + morph_helper = MorphologyHelper() + cluster_morph = morph_helper.load_morphology(cluster_path) + systems = dict(morph_helper.load_cluster_systems(cluster_morph)) + + for system_path, deployment_name, deployment_config in \ + morph_helper.iterate_cluster_deployments(cluster_morph): + + system_morph = systems[system_path] + # We can only test systems in KVM that have a BSP + if not any('bsp' in si['morph'] for si in system_morph['strata']): + continue + + # We can only test systems in KVM that we have a host for + if system_morph['arch'] not in deployment_hosts: + continue + host_machine = deployment_hosts[system_morph['arch']] + deployment = Deployment(cluster_path, deployment_name, + deployment_config, host_machine) + + instance = deployment.deploy() + try: + self._run_tests(instance, system_path, system_morph, + build_test_config, morph_helper, systems) + finally: + instance.delete() + + def process_args(self, args): + """Process the command line args and kick off the builds/tests""" + if self.settings['build-ref-prefix'] is None: + self.settings['build-ref-prefix'] = ( + os.path.join(self.settings['trove-id'], 'builds')) + for setting in ('deployment-host', 'trove-host', + 'trove-id', 'build-ref-prefix'): + self.settings.require(setting) + + deployment_hosts = {} + for host_config in self.settings['deployment-host']: + arch, address = host_config.split(':', 1) + user, address = address.split('@', 1) + address, disk_path = address.split(':', 1) + if user == '': + user = 'root' + # TODO: Don't assume root is the user with deploy access + deployment_hosts[arch] = VMHost(user, address, disk_path) + + build_test_config = (self.settings['trove-host'], + self.settings['trove-id'], + self.settings['build-ref-prefix']) + + if len(args) != 1: + raise cliapp.AppException('Usage: release-test CLUSTER') + cluster_path = morphlib.util.sanitise_morphology_path(args[0]) + self.deploy_and_test_systems(cluster_path, deployment_hosts, + build_test_config) + + +if __name__ == '__main__': + ReleaseApp().run() diff --git a/old/scripts/release-test-os b/old/scripts/release-test-os new file mode 100755 index 00000000..06e01daf --- /dev/null +++ b/old/scripts/release-test-os @@ -0,0 +1,526 @@ +#!/usr/bin/env python +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''release-test + +This script deploys the set of systems in the cluster morphology it is +instructed to read, to test that they work correctly. + +''' + +import cliapp +import os +import pipes +import shlex +import shutil +import socket +import tempfile +import time +import uuid + +import morphlib + + +class NovaList: + def __init__(self): + self.output = [] + self.lines = [] + self.instance = [] + + def update(self): + self.output = cliapp.runcmd(['nova', 'list']) + self.lines = self.output.split('\n') + self.lines = self.lines[3:-2] + + def get_nova_details_for_instance(self, name): + self.update() + + for line in self.lines: + entries = line.split('|') + stripped_line = [entry.strip() for entry in entries] + if stripped_line.count(name) == 1: + self.instance = stripped_line + + def get_nova_state_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + return self.instance[3] + + def get_nova_ip_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + + if self.get_nova_state_for_instance(name) != 'ACTIVE': + return + + return self.instance[6] + + def get_nova_ip_for_instance_timeout(self, name, timeout=120): + start_time = time.time() + + while self.get_nova_state_for_instance(name) != 'ACTIVE': + + if time.time() > start_time + timeout: + print "%s not ACTIVE after %i seconds" % (name, timeout) + return + + time.sleep(1) + + ip_addr = self.get_nova_ip_for_instance(name) + if not ip_addr: + return + + if ip_addr.count('=') == 0: + return + + ip_addr = ip_addr[ip_addr.find('=') + 1:] + + if ip_addr.count(',') == 0: + return ip_addr + + return ip_addr[:ip_addr.find(',')] + + + +class MorphologyHelper(object): + + def __init__(self): + self.sb = sb = morphlib.sysbranchdir.open_from_within('.') + defs_repo_path = sb.get_git_directory_name(sb.root_repository_url) + self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) + + def load_morphology(self, path): + text = self.finder.read_file(path) + return self.loader.load_from_string(text) + + @classmethod + def iterate_systems(cls, systems_list): + for system in systems_list: + yield morphlib.util.sanitise_morphology_path(system['morph']) + if 'subsystems' in system: + for subsystem in cls.iterate_systems(system['subsystems']): + yield subsystem + + def iterate_cluster_deployments(cls, cluster_morph): + for system in cluster_morph['systems']: + path = morphlib.util.sanitise_morphology_path(system['morph']) + defaults = system.get('deploy-defaults', {}) + for name, options in system['deploy'].iteritems(): + config = dict(defaults) + config.update(options) + yield path, name, config + + def load_cluster_systems(self, cluster_morph): + for system_path in set(self.iterate_systems(cluster_morph['systems'])): + system_morph = self.load_morphology(system_path) + yield system_path, system_morph + + +class TimeoutError(cliapp.AppException): + + """Error to be raised when a connection waits too long""" + + def __init__(self, msg): + super(TimeoutError, self).__init__(msg) + + +class VMHost(object): + + def __init__(self, user, address, disk_path): + self.user = user + self.address = address + self.disk_path = disk_path + + @property + def ssh_host(self): + return '{user}@{address}'.format(user=self.user, address=self.address) + + def runcmd(self, *args, **kwargs): + cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) + + +class DeployedSystemInstance(object): + + def __init__(self, deployment, config, host_machine, vm_id, rootfs_path, + ip_addr, hostname): + self.deployment = deployment + self.config = config + self.ip_address = ip_addr + self.host_machine = host_machine + self.vm_id = vm_id + self.rootfs_path = rootfs_path + self.hostname = hostname + + @property + def ssh_host(self): + # TODO: Stop assuming we ssh into test instances as root + return 'root@{host}'.format(host=self.ip_address) + + def runcmd(self, argv, chdir='.', **kwargs): + ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] + cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] + cmd += argv + ssh_cmd.append(' '.join(map(pipes.quote, cmd))) + return cliapp.runcmd(ssh_cmd, **kwargs) + + def _wait_for_dhcp(self, timeout): + '''Block until given hostname resolves successfully. + + Raises TimeoutError if the hostname has not appeared in 'timeout' + seconds. + + ''' + start_time = time.time() + while True: + try: + socket.gethostbyname(self.ip_address) + return + except socket.gaierror: + pass + if time.time() > start_time + timeout: + raise TimeoutError("Host %s did not appear after %i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_ssh(self, timeout): + """Wait until the deployed VM is responding via SSH""" + start_time = time.time() + while True: + try: + self.runcmd(['true'], stdin=None, stdout=None, stderr=None) + return + except cliapp.AppException: + # TODO: Stop assuming the ssh part of the command is what failed + if time.time() > start_time + timeout: + raise TimeoutError("%s sshd did not start after %i seconds" + % (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_cloud_init(self, timeout): + """Wait until cloud init has resized the disc""" + start_time = time.time() + while True: + try: + out = self.runcmd(['sh', '-c', + 'test -e "$1" && echo exists || echo does not exist', + '-', + '/root/cloud-init-finished']) + except: + import traceback + traceback.print_exc() + raise + if out.strip() == 'exists': + return + if time.time() > start_time + timeout: + raise TimeoutError("Disc size not increased after %i seconds" + % (timeout)) + time.sleep(3) + + def wait_until_online(self, timeout=120): + self._wait_for_dhcp(timeout) + self._wait_for_ssh(timeout) + self._wait_for_cloud_init(timeout) + print "Test system %s ready to run tests." % (self.hostname) + + def delete(self): + # Stop and remove VM + print "Deleting %s test instance" % (self.hostname) + try: + cliapp.runcmd(['nova', 'delete', self.hostname]) + except cliapp.AppException as e: + # TODO: Stop assuming that delete failed because the instance + # wasn't running + print "- Failed" + pass + print "Deleting %s test disc image" % (self.hostname) + try: + cliapp.runcmd(['nova', 'image-delete', self.hostname]) + except cliapp.AppException as e: + # TODO: Stop assuming that image-delete failed because it was + # already removed + print "- Failed" + pass + + +class Deployment(object): + + def __init__(self, cluster_path, name, deployment_config, + host_machine, net_id): + self.cluster_path = cluster_path + self.name = name + self.deployment_config = deployment_config + self.host_machine = host_machine + self.net_id = net_id + + @staticmethod + def _ssh_host_key_exists(hostname): + """Check if an ssh host key exists in known_hosts""" + if not os.path.exists('/root/.ssh/known_hosts'): + return False + with open('/root/.ssh/known_hosts', 'r') as known_hosts: + return any(line.startswith(hostname) for line in known_hosts) + + def _update_known_hosts(self): + if not self._ssh_host_key_exists(self.host_machine.address): + with open('/root/.ssh/known_hosts', 'a') as known_hosts: + cliapp.runcmd(['ssh-keyscan', self.host_machine.address], + stdout=known_hosts) + + @staticmethod + def _generate_sshkey_config(tempdir, config): + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write('0040700 0 0 /root/.ssh\n') + f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') + authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') + os.makedirs(os.path.dirname(authkeys)) + with open(authkeys, 'w') as auth_f: + with open('/root/.ssh/id_rsa.pub', 'r') as key_f: + shutil.copyfileobj(key_f, auth_f) + + install_files = shlex.split(config.get('INSTALL_FILES', '')) + install_files.append(manifest) + yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) + + def deploy(self): + self._update_known_hosts() + + hostname = str(uuid.uuid4()) + vm_id = hostname + image_base = self.host_machine.disk_path + rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, + hostname=hostname) + loc = 'http://{ssh_host}:5000/v2.0'.format( + ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) + + options = { + 'type': 'openstack', + 'location': loc, + 'HOSTNAME': hostname, + 'DISK_SIZE': '5G', + 'RAM_SIZE': '2G', + 'VERSION_LABEL': 'release-test', + 'OPENSTACK_USER': os.environ['OS_USERNAME'], + 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'], + 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'], + 'OPENSTACK_IMAGENAME': hostname, + 'CLOUD_INIT': 'yes', + 'KERNEL_ARGS': 'console=tty0 console=ttyS0', + } + + tempdir = tempfile.mkdtemp() + try: + options.update( + self._generate_sshkey_config(tempdir, + self.deployment_config)) + + # Deploy the image to openstack + args = ['morph', 'deploy', self.cluster_path, self.name] + for k, v in options.iteritems(): + args.append('%s.%s=%s' % (self.name, k, v)) + cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) + + config = dict(self.deployment_config) + config.update(options) + + # Boot an instance from the image + args = ['nova', 'boot', + '--flavor', 'm1.medium', + '--image', hostname, + '--user-data', '/usr/lib/mason/os-init-script', + '--nic', "net-id=%s" % (self.net_id), + hostname] + output = cliapp.runcmd(args) + + # Print nova boot output, with adminPass line removed + output_lines = output.split('\n') + for line in output_lines: + if line.find('adminPass') != -1: + password_line = line + output_lines.remove(password_line) + output = '\n'.join(output_lines) + print output + + # Get ip address from nova list + nl = NovaList() + ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) + print "IP address for instance %s: %s" % (hostname, ip_addr) + + return DeployedSystemInstance(self, config, self.host_machine, + vm_id, rootpath, ip_addr, hostname) + finally: + shutil.rmtree(tempdir) + + +class ReleaseApp(cliapp.Application): + + """Cliapp application which handles automatic builds and tests""" + + def add_settings(self): + """Add the command line options needed""" + group_main = 'Program Options' + self.settings.string_list(['deployment-host'], + 'ARCH:HOST:PATH that VMs can be deployed to', + default=None, + group=group_main) + self.settings.string(['trove-host'], + 'Address of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['trove-id'], + 'ID of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['build-ref-prefix'], + 'Prefix of build branches for test systems', + default=None, + group=group_main) + self.settings.string(['net-id'], + 'Openstack network ID', + default=None, + group=group_main) + + @staticmethod + def _run_tests(instance, system_path, system_morph, + (trove_host, trove_id, build_ref_prefix), + morph_helper, systems): + instance.wait_until_online() + + tests = [] + def baserock_build_test(instance): + instance.runcmd(['git', 'config', '--global', 'user.name', + 'Test Instance of %s' % instance.deployment.name]) + instance.runcmd(['git', 'config', '--global', 'user.email', + 'ci-test@%s' % instance.config['HOSTNAME']]) + instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', + '/src/tmp']) + def morph_cmd(*args, **kwargs): + # TODO: decide whether to use cached artifacts or not by + # adding --artifact-cache-server= --cache-server= + argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', + '--tempdir=/src/tmp', '--log-max=100M', + '--trove-host', trove_host, '--trove-id', trove_id, + '--build-ref-prefix', build_ref_prefix] + argv.extend(args) + instance.runcmd(argv, **kwargs) + + repo = morph_helper.sb.root_repository_url + ref = morph_helper.defs_repo.HEAD + sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref) + morph_cmd('init', '/src/ws') + chdir = '/src/ws' + + morph_cmd('checkout', repo, ref, chdir=chdir) + # TODO: Add a morph subcommand that gives the path to the root repository. + repo_path = os.path.relpath( + morph_helper.sb.get_git_directory_name(repo), + morph_helper.sb.root_directory) + chdir = os.path.join(chdir, ref, repo_path) + + instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) + print 'Building test systems for {sys}'.format(sys=system_path) + for to_build_path, to_build_morph in systems.iteritems(): + if to_build_morph['arch'] == system_morph['arch']: + print 'Test building {path}'.format(path=to_build_path) + morph_cmd('build', to_build_path, chdir=chdir, + stdin=None, stdout=None, stderr=None) + print 'Finished Building test systems' + + def python_smoke_test(instance): + instance.runcmd(['python', '-c', 'print "Hello World"']) + + # TODO: Come up with a better way of determining which tests to run + if 'devel' in system_path: + tests.append(baserock_build_test) + else: + tests.append(python_smoke_test) + + for test in tests: + test(instance) + + def deploy_and_test_systems(self, cluster_path, + deployment_hosts, build_test_config, + net_id): + """Run the deployments and tests""" + + version = 'release-test' + + morph_helper = MorphologyHelper() + cluster_morph = morph_helper.load_morphology(cluster_path) + systems = dict(morph_helper.load_cluster_systems(cluster_morph)) + + for system_path, deployment_name, deployment_config in \ + morph_helper.iterate_cluster_deployments(cluster_morph): + + system_morph = systems[system_path] + # We can only test systems in KVM that have a BSP + if not any('bsp' in si['morph'] for si in system_morph['strata']): + continue + + # We can only test systems in KVM that we have a host for + if system_morph['arch'] not in deployment_hosts: + continue + host_machine = deployment_hosts[system_morph['arch']] + deployment = Deployment(cluster_path, deployment_name, + deployment_config, host_machine, + net_id) + + instance = deployment.deploy() + try: + self._run_tests(instance, system_path, system_morph, + build_test_config, morph_helper, systems) + finally: + instance.delete() + + def process_args(self, args): + """Process the command line args and kick off the builds/tests""" + if self.settings['build-ref-prefix'] is None: + self.settings['build-ref-prefix'] = ( + os.path.join(self.settings['trove-id'], 'builds')) + for setting in ('deployment-host', 'trove-host', + 'trove-id', 'build-ref-prefix', 'net-id'): + self.settings.require(setting) + + deployment_hosts = {} + for host_config in self.settings['deployment-host']: + arch, address = host_config.split(':', 1) + user, address = address.split('@', 1) + address, disk_path = address.split(':', 1) + if user == '': + user = 'root' + # TODO: Don't assume root is the user with deploy access + deployment_hosts[arch] = VMHost(user, address, disk_path) + + build_test_config = (self.settings['trove-host'], + self.settings['trove-id'], + self.settings['build-ref-prefix']) + + if len(args) != 1: + raise cliapp.AppException('Usage: release-test CLUSTER') + cluster_path = morphlib.util.sanitise_morphology_path(args[0]) + self.deploy_and_test_systems(cluster_path, deployment_hosts, + build_test_config, + self.settings['net-id']) + + +if __name__ == '__main__': + ReleaseApp().run() diff --git a/old/scripts/release-upload b/old/scripts/release-upload new file mode 100755 index 00000000..f8b3337c --- /dev/null +++ b/old/scripts/release-upload @@ -0,0 +1,473 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Upload and publish Baserock binaries for a release. + +This utility is used for the Baserock release process. See +http://wiki.baserock.org/guides/release-process/ for details on the +release process. + +This utility uploads two sets of binaries: + +* The build artifacts (built chunks and strata) used to construct the + systems being released. The systems are found in `release.morph` and + the artifacts from the Trove used to prepare the release. They get + uploaded to a public Trove (by default git.baserock.org). If they're + the same Trove, then nothing happens. + +* The released system images (disk images, tar archives, etc) + specified in `release.morph` get uploaded to a download server (by + default download.baserock.org). + +''' + + +import json +import logging +import os +import pwd +import shutil +import sys +import urllib +import urllib2 +import urlparse + +import cliapp +import yaml + +import morphlib + +class ReleaseUploader(cliapp.Application): + + def add_settings(self): + group = 'Release upload settings' + + local_username = self.get_local_username() + + self.settings.string( + ['build-trove-host'], + 'get build artifacts from Trove at ADDRESS', + metavar='ADDRESS', + group=group) + + self.settings.string( + ['public-trove-host'], + 'publish build artifacts on Trove at ADDRESS', + metavar='ADDRESS', + default='git.baserock.org', + group=group) + + self.settings.string( + ['public-trove-username'], + 'log into public trove as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['public-trove-artifact-dir'], + 'put published artifacts into DIR', + metavar='DIR', + default='/home/cache/artifacts', + group=group) + + self.settings.string( + ['release-artifact-dir'], + 'get release artifacts from DIR (all files from there)', + metavar='DIR', + default='.', + group=group) + + self.settings.string( + ['download-server-address'], + 'publish release artifacts on server at ADDRESS', + metavar='ADDRESS', + default='download.baserock.org', + group=group) + + self.settings.string( + ['download-server-username'], + 'log into download server as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['download-server-private-dir'], + 'use DIR as the temporary location for uploaded release ' + 'artifacts', + metavar='DIR', + default='/srv/download.baserock.org/baserock/.publish-temp', + group=group) + + self.settings.string( + ['download-server-public-dir'], + 'put published release artifacts in DIR', + metavar='DIR', + default='/srv/download.baserock.org/baserock', + group=group) + + self.settings.string( + ['local-build-artifacts-dir'], + 'keep build artifacts to be uploaded temporarily in DIR', + metavar='DIR', + default='build-artifacts', + group=group) + + self.settings.string( + ['morph-cmd'], + 'run FILE to invoke morph', + metavar='FILE', + default='morph', + group=group) + + self.settings.string_list( + ['arch'], + 'Upload files from morphologies of ARCH', + metavar='ARCH', + default=[], + group=group) + + self.settings.boolean( + ['upload-build-artifacts'], + 'upload build artifacts?', + default=True) + + self.settings.boolean( + ['upload-release-artifacts'], + 'upload release artifacts (disk images etc)?', + default=True) + + def get_local_username(self): + uid = os.getuid() + return pwd.getpwuid(uid)[0] + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Usage: release-upload CLUSTER') + cluster_morphology_path = args[0] + self.status(msg='Uploading and publishing Baserock release') + + if self.settings['upload-build-artifacts']: + self.publish_build_artifacts(cluster_morphology_path) + else: + self.status( + msg='Not uploading build artifacts ' + '(upload-build-artifacts set to false') + + if self.settings['upload-release-artifacts']: + self.publish_release_artifacts() + else: + self.status( + msg='Not uploading release artifacts ' + '(upload-release-artifacts set to false') + + def publish_build_artifacts(self, cluster_morphology_path): + publisher = BuildArtifactPublisher(self.settings, self.status) + publisher.publish_build_artifacts(cluster_morphology_path) + self.status(msg='Build artifacts have been published') + + def publish_release_artifacts(self): + publisher = ReleaseArtifactPublisher(self.settings, self.status) + publisher.publish_release_artifacts() + self.status(msg='Release artifacts have been published') + + def status(self, msg, **kwargs): + formatted = msg.format(**kwargs) + logging.info(formatted) + sys.stdout.write(formatted + '\n') + sys.stdout.flush() + + +class BuildArtifactPublisher(object): + + '''Publish build artifacts related to the release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_build_artifacts(self, cluster_path): + artifact_basenames = self.list_build_artifacts_for_release(cluster_path) + self.status( + msg='Found {count} build artifact files in release', + count=len(artifact_basenames)) + + to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( + artifact_basenames) + + logging.debug('List of artifacts (basenames) to upload (without already uploaded):') + for i, basename in enumerate(to_be_uploaded): + logging.debug(' {0}: {1}'.format(i, basename)) + logging.debug('End of artifact list (to_be_uploaded)') + + self.status( + msg='Need to fetch locally, then upload {count} build artifacts', + count=len(to_be_uploaded)) + + self.upload_build_artifacts_to_public_trove(to_be_uploaded) + + def list_build_artifacts_for_release(self, cluster_morphology_path): + self.status(msg='Find build artifacts included in release') + + # FIXME: These are hardcoded for simplicity. They would be + # possible to deduce automatically from the workspace, but + # that can happen later. + repo = 'file://%s' % os.path.abspath('.') + ref = 'HEAD' + + argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', + '--repo', repo, '--ref', ref] + argv += self.find_system_morphologies(cluster_morphology_path) + output = cliapp.runcmd(argv) + basenames = output.splitlines() + logging.debug('List of build artifacts in release:') + for basename in basenames: + logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') + + return basenames + + def find_system_morphologies(self, cluster_morphology_path): + cluster = self.load_cluster_morphology(cluster_morphology_path) + system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) + if self.settings['arch']: + system_dicts = self.choose_systems_for_wanted_architectures( + system_dicts, self.settings['arch']) + return [sd['morph'] for sd in system_dicts] + + def load_cluster_morphology(self, pathname): + with open(pathname) as f: + return yaml.load(f) + + def find_systems_in_parsed_cluster_morphology(self, cluster): + return cluster['systems'] + + def choose_systems_for_wanted_architectures(self, system_dicts, archs): + return [ + sd + for sd in system_dicts + if self.system_is_for_wanted_arch(sd, archs)] + + def system_is_for_wanted_arch(self, system_dict, archs): + morph = self.load_system_morphology(system_dict) + return morph['arch'] in archs + + def load_system_morphology(self, system_dict): + pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) + return self.load_morphology_from_named_file(pathname) + + def load_morphology_from_named_file(self, pathname): + finder = self.get_morphology_finder_for_root_repository() + morphology_text = finder.read_file(pathname) + loader = morphlib.morphloader.MorphologyLoader() + return loader.load_from_string(morphology_text) + + def get_morphology_finder_for_root_repository(self): + definitions_repo = morphlib.definitions_repo.open( + '.', search_for_root=True) + return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) + + def filter_away_build_artifacts_on_public_trove(self, basenames): + result = [] + logging.debug('Filtering away already existing artifacts:') + for basename, exists in self.query_public_trove_for_artifacts(basenames): + logging.debug(' {0}: {1}'.format(basename, exists)) + if not exists: + result.append(basename) + logging.debug('End of filtering away') + return result + + def query_public_trove_for_artifacts(self, basenames): + host = self.settings['public-trove-host'] + + # FIXME: This could use + # contextlib.closing(urllib2.urlopen(url, data=data) instead + # of explicit closing. + url = 'http://{host}:8080/1.0/artifacts'.format(host=host) + data = json.dumps(basenames) + f = urllib2.urlopen(url, data=data) + obj = json.load(f) + return obj.items() + + def upload_build_artifacts_to_public_trove(self, basenames): + self.download_artifacts_locally(basenames) + self.upload_artifacts_to_public_trove(basenames) + + def download_artifacts_locally(self, basenames): + dirname = self.settings['local-build-artifacts-dir'] + self.create_directory_if_missing(dirname) + for i, basename in enumerate(basenames): + url = self.construct_artifact_url(basename) + pathname = os.path.join(dirname, basename) + if not os.path.exists(pathname): + self.status( + msg='Downloading {i}/{total} {basename}', + basename=repr(basename), i=i, total=len(basenames)) + self.download_from_url(url, dirname, pathname) + + def create_directory_if_missing(self, dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + + def construct_artifact_url(self, basename): + scheme = 'http' + netloc = '{host}:8080'.format(host=self.settings['build-trove-host']) + path = '/1.0/artifacts' + query = 'filename={0}'.format(urllib.quote_plus(basename)) + fragment = '' + components = (scheme, netloc, path, query, fragment) + return urlparse.urlunsplit(components) + + def download_from_url(self, url, dirname, pathname): + logging.info( + 'Downloading {url} to {pathname}'.format( + url=url, pathname=pathname)) + with open(pathname, 'wb') as output: + try: + incoming = urllib2.urlopen(url) + shutil.copyfileobj(incoming, output) + incoming.close() + except urllib2.HTTPError as e: + if pathname.endswith('.meta'): + return + self.status( + msg="ERROR: Can't download {url}: {explanation}", + url=url, + explanation=str(e)) + os.remove(pathname) + raise + + def upload_artifacts_to_public_trove(self, basenames): + self.status( + msg='Upload build artifacts to {trove}', + trove=self.settings['public-trove-host']) + rsync_files_to_server( + self.settings['local-build-artifacts-dir'], + basenames, + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir']) + set_permissions_on_server( + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir'], + basenames) + +class ReleaseArtifactPublisher(object): + + '''Publish release artifacts for a release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_release_artifacts(self): + files = self.list_release_artifacts() + if files: + self.upload_release_artifacts_to_private_dir(files) + self.move_release_artifacts_to_public_dir(files) + self.create_symlinks_to_new_release_artifacts(files) + + def list_release_artifacts(self): + self.status(msg='Find release artifacts to publish') + return os.listdir(self.settings['release-artifact-dir']) + + def upload_release_artifacts_to_private_dir(self, files): + self.status(msg='Upload release artifacts to private directory') + path = self.settings['download-server-private-dir'] + self.create_directory_on_download_server(path) + self.rsync_files_to_download_server(files, path) + + def create_directory_on_download_server(self, path): + user = self.settings['download-server-username'] + host = self.settings['download-server-address'] + self.status(msg='Create {host}:{path}', host=host, path=path) + target = '{user}@{host}'.format(user=user, host=host) + cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) + + def rsync_files_to_download_server(self, files, path): + self.status(msg='Upload release artifacts to download server') + rsync_files_to_server( + self.settings['release-artifact-dir'], + files, + self.settings['download-server-username'], + self.settings['download-server-address'], + path) + set_permissions_on_server( + self.settings['download-server-username'], + self.settings['download-server-address'], + path, + files) + + def move_release_artifacts_to_public_dir(self, files): + self.status(msg='Move release artifacts to public directory') + private_dir = self.settings['download-server-private-dir'] + public_dir = self.settings['download-server-public-dir'] + self.create_directory_on_download_server(public_dir) + + # Move just the contents of the private dir, not the dir + # itself (-mindepth). Avoid overwriting existing files (mv + # -n). + argv = ['find', private_dir, '-mindepth', '1', + '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] + + target = '{user}@{host}'.format( + user=self.settings['download-server-username'], + host=self.settings['download-server-address']) + cliapp.ssh_runcmd(target, argv) + + def create_symlinks_to_new_release_artifacts(self, files): + self.status(msg='FIXME: Create symlinks to new releas artifacts') + + +def rsync_files_to_server( + source_dir, source_filenames, user, host, target_dir): + + if not source_filenames: + return + + argv = [ + 'rsync', + '-a', + '--progress', + '--partial', + '--human-readable', + '--sparse', + '--protect-args', + '-0', + '--files-from=-', + source_dir, + '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), + ] + + files_list = '\0'.join(filename for filename in source_filenames) + cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) + + +def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return + target = '{user}@{host}'.format(user=user, host=host) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) + + +ReleaseUploader(description=__doc__).run() diff --git a/old/scripts/release-upload.test.conf b/old/scripts/release-upload.test.conf new file mode 100644 index 00000000..13227983 --- /dev/null +++ b/old/scripts/release-upload.test.conf @@ -0,0 +1,10 @@ +[config] +download-server-address = localhost +download-server-private-dir = /tmp/private +download-server-public-dir = /tmp/public +build-trove-host = ct-mcr-1.ducie.codethink.co.uk +public-trove-host = localhost +public-trove-username = root +public-trove-artifact-dir = /tmp/artifacts +release-artifact-dir = t.release-files +morph-cmd = /home/root/git-morph diff --git a/old/scripts/scriptslib.py b/old/scripts/scriptslib.py new file mode 100644 index 00000000..53c6ca8e --- /dev/null +++ b/old/scripts/scriptslib.py @@ -0,0 +1,156 @@ +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Small library of useful things for the scripts that live here. + +import yaml +import subprocess +import os +import sys + +aliases = { + 'baserock:': 'git://%(trove)s/baserock/', + 'freedesktop:': 'git://anongit.freedesktop.org/', + 'github:': 'git://github.com/', + 'gnome:': 'git://git.gnome.org/', + 'upstream:': 'git://%(trove)s/delta/' +} + +def parse_repo_alias(repo, trove_host='git.baserock.org'): + global aliases + remote = repo[:repo.find(':') + 1] + aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()} + try: + return repo.replace(remote, aliases[remote]) + except KeyError as e: + raise Exception("Unknown repo-alias \"%s\"" % repo) + +def definitions_root(): + return subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"]).strip() + +def load_yaml_file(yaml_file): + with open(yaml_file, 'r') as f: + return yaml.safe_load(f) + + +class BaserockMeta(object): + '''An object representing Baserock metadata contained in a Baserock + system image, for available metadata formats''' + + def __init__(self): + self.metas = {} + + def get_each(self): + '''Yield an iterable for the whole list of metas''' + for key in self.metas: + yield self.metas[key] + + def get_name(self, name): + '''Yield an iterable of metadata matched by name, e.g. `bash`''' + for key in self.metas: + if self.metas[key]['source-name'] == name: + yield self.metas[key] + + def import_meta(self, meta_text): + importers = (self.import_meta_ybd, + self.import_meta_morph) + + for i in importers: + try: + i(meta_text) + return + except (KeyError, Exception) as err: + pass + + # Shouldn't get here + sys.stderr.write('Metadata format not recognised.\n' + 'Error:\n') + raise err + + def import_meta_morph(self, meta_text): + self._add_meta(yaml.load(meta_text)) + + def import_meta_ybd(self, meta_text): + source = yaml.load(meta_text) + + null = '0' * 32 + + if 'configuration' in source: + # This is the deployment metadata, ignore + return + elif 'repo' not in source: + kind = 'stratum' + contents = 'components' + source['repo'] = 'upstream:definitions' + source['ref'] = null # No ref info + else: + kind = 'chunk' + contents = 'components' + + repo = parse_repo_alias(source['repo']) + source_name = '-'.join( + source['products'][0]['artifact'].split('-')[:-1]) + + # Needed until YBD provides cache-key in metadata + if not 'cache-key' in source: + source['cache-key'] = null + + for product in source['products']: + + self._add_meta({ + 'kind': kind, + 'source-name': source_name, + 'artifact-name': product['artifact'], + 'contents': product[contents], + 'repo': repo, + 'repo-alias': source['repo'], + 'sha1': source['ref'], + 'original_ref': source['ref'], + 'cache-key': source['cache-key'] + }) + + def _add_meta(self, meta_dict): + '''Validate and add a meta''' + + ignore = ('configuration', + 'system-artifact-name') + + for i in ignore: + if i in meta_dict: + return + + required_fields = ('repo', 'sha1', 'contents') + for f in required_fields: + if not f in meta_dict: + raise Exception('Metadata format not recognised, no ' + 'value for \'%s\'. Data: \'%s\''% (f, str(meta_dict))) + + self.metas[meta_dict['artifact-name']] = meta_dict + + +def meta_load_from_dir(meta_dir_path): + '''Read Baserock metadata from a directory''' + + files = [f for f in os.listdir(meta_dir_path) + if os.path.isfile(os.path.join(meta_dir_path, f))] + + meta = BaserockMeta() + for f in files: + if f.endswith('.meta'): + meta.import_meta( + open(os.path.join(meta_dir_path, f), 'r').read()) + + return meta diff --git a/old/scripts/yaml-jsonschema b/old/scripts/yaml-jsonschema new file mode 100755 index 00000000..64f52a79 --- /dev/null +++ b/old/scripts/yaml-jsonschema @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A tool to validate YAML files against the JSON-Schema schemas. + +This wraps Python `jsonschema` module so that YAML schemas can be understood +and YAML data can be validated. + +Usage: yaml-jsonschema SCHEMA INPUT1 [INPUT2, ...] + +''' + + +import jsonschema +import yaml + +import sys + + +schema_file = sys.argv[1] +input_files = sys.argv[2:] + + +with open(schema_file) as f: + schema = yaml.load(f) + + +for input_file in input_files: + with open(input_file) as f: + data = yaml.load(f) + + try: + jsonschema.validate(data, schema) + print("%s: valid" % input_file) + except jsonschema.ValidationError as e: + # Print 'e' instead of 'e.message' for more information! + print("%s: %s" % (input_file, e.message)) |