From 7df7f3b427739ff7d69da2ba218da0124822892c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 26 Nov 2017 23:39:48 +0000 Subject: Remove all .morph files and files from the old format --- scripts/baserock-release-metadata | 66 ----- scripts/check-unpetrify-refs.py | 76 ----- scripts/cycle.sh | 61 ---- scripts/licensecheck.pl | 604 -------------------------------------- scripts/licensecheck.py | 201 ------------- scripts/organize-morphologies.py | 255 ---------------- scripts/release-build | 192 ------------ scripts/release-build.test.conf | 6 - scripts/release-test | 400 ------------------------- scripts/release-test-os | 526 --------------------------------- scripts/release-upload | 473 ----------------------------- scripts/release-upload.test.conf | 10 - scripts/scriptslib.py | 156 ---------- scripts/yaml-jsonschema | 50 ---- 14 files changed, 3076 deletions(-) delete mode 100755 scripts/baserock-release-metadata delete mode 100755 scripts/check-unpetrify-refs.py delete mode 100755 scripts/cycle.sh delete mode 100644 scripts/licensecheck.pl delete mode 100755 scripts/licensecheck.py delete mode 100755 scripts/organize-morphologies.py delete mode 100755 scripts/release-build delete mode 100644 scripts/release-build.test.conf delete mode 100755 scripts/release-test delete mode 100755 scripts/release-test-os delete mode 100755 scripts/release-upload delete mode 100644 scripts/release-upload.test.conf delete mode 100644 scripts/scriptslib.py delete mode 100755 scripts/yaml-jsonschema (limited to 'scripts') diff --git a/scripts/baserock-release-metadata b/scripts/baserock-release-metadata deleted file mode 100755 index 1b429671..00000000 --- a/scripts/baserock-release-metadata +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., - - -'''baserock-release-metadata - -Generates a YAML document with some simple metadata that should be included -in release binaries - -This expects to be run inside the definitions repo so that it can figure -out the Git commit being built. - -''' - -import collections -import subprocess - - -def git_show_commit(): - return subprocess.check_output( - ['git', 'rev-parse', 'HEAD']).decode('unicode-escape').strip() - - -def buildstream_version(): - return subprocess.check_output( - ['bst', '--version']).decode('unicode-escape').strip() - - -def main(): - repo = "https://www.gitlab.com/baserock/definitions" - commit = git_show_commit() - build_tool = "Buildstream" - # This marks the version of BuildStream that checked out the sysroot - # artifact; the artifact may have pulled it from a cache though so - # we don't know what version of BuildStream actually built anything. - build_tool_version = buildstream_version() - - # Here we produce a YAML document with specific ordering and formatting - # to ensure readability. - print("description: |\n" - " This sysroot was produced by the Baserock project.\n" - " See for more information.\n") - print("project-repo: %s" % repo) - print("project-commit: %s" % commit) - print() - print("build-tool: %s" % build_tool) - print("build-tool-version: %s" % build_tool_version) - - -try: - main() -except RuntimeError as e: - sys.stderr.write("%s\n" % e) - sys.exit(1) diff --git a/scripts/check-unpetrify-refs.py b/scripts/check-unpetrify-refs.py deleted file mode 100755 index c70b680d..00000000 --- a/scripts/check-unpetrify-refs.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import glob -import argparse -import subprocess - -import scriptslib - - -''' -Script for checking unpetrify-refs in strata. - -Without args this script will check everything in strata/, or each stratum -given on the command-line and will print on stdout whether a chunk has -a missing or non-existent unpetrify-ref and if it fails to check the remote -(missing repo). -''' - -strata_dir = "strata" - -def ref_exists(remote, ref): - output = subprocess.check_output( - ["git", "ls-remote", remote, str(ref)], - stderr=subprocess.STDOUT).strip() - return True if output else False - -def main(): - parser = argparse.ArgumentParser( - description="Sanity checks unpetrify-refs in Baserock strata") - parser.add_argument("--trove-host", default="git.baserock.org", - help="Trove host to map repo aliases to") - parser.add_argument("strata", nargs="*", metavar="STRATA", - help="The strata to check (checks all by default)") - args = parser.parse_args() - - if args.strata: - strata = args.strata - else: - strata_path = os.path.join(scriptslib.definitions_root(), strata_dir) - strata = glob.glob("%s/*.morph" % strata_path) - - for stratum in strata: - path = os.path.relpath(stratum) - morphology = scriptslib.load_yaml_file(stratum) - for chunk in morphology['chunks']: - unpetrify_ref = chunk.get("unpetrify-ref") - if not unpetrify_ref: - print ("%s: '%s' has no unpetrify-ref!" % - (path, chunk['name'])) - continue - remote = scriptslib.parse_repo_alias(chunk['repo'], args.trove_host) - try: - if not ref_exists(remote, unpetrify_ref): - print ("%s: unpetrify-ref for '%s' is not present on the " - "remote (%s)!" % (path, chunk['name'], remote)) - except subprocess.CalledProcessError as e: - print ("%s: failed to ls-remote (%s) for chunk '%s':\n%s" % - (path, remote, chunk['name'], e.output.strip())) - -if __name__ == "__main__": - main() diff --git a/scripts/cycle.sh b/scripts/cycle.sh deleted file mode 100755 index c0e2aa67..00000000 --- a/scripts/cycle.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -usage() { - echo "Usage: cycle.sh some-system some-cluster [newversion]" - echo - echo "This builds and deploys the current checked out version of" - echo "some-system, applying it as a self-upgrade to the system you" - echo "are working in, using configuration from some-cluster." - echo "The upgrade is labelled TEST by default, or [newversion] if" - echo "specified, and is set to be the default for next boot." -} - -if [ -z "$1" ] || [ -z "$2" ] || [ ! -z "$4" ] ; then - usage - exit 1 -fi - -newversion=TEST -if [ ! -z "$3" ] ; then - newversion=$3 - if (echo "$newversion" | grep ' ' > /dev/null 2>&1) ; then - echo 'Version label must not contain spaces.' - exit 1 - fi -fi - -if system-version-manager get-running | grep -q "^$newversion$"; then - echo "You are currently running the $newversion system." - echo "Maybe you want to boot into a different system version?" - exit 1 -fi - -set -e -set -v - -runningversion=`system-version-manager get-running` -system-version-manager set-default $runningversion -if system-version-manager list | grep -q "^$newversion$"; then - system-version-manager remove $newversion -fi - -morph gc -morph build "$1" - -sed -i "s|^- morph: .*$|- morph: $1|" "$2" -morph deploy --upgrade "$2" self.HOSTNAME=$(hostname) self.VERSION_LABEL=$newversion -system-version-manager list diff --git a/scripts/licensecheck.pl b/scripts/licensecheck.pl deleted file mode 100644 index 5b6d0d33..00000000 --- a/scripts/licensecheck.pl +++ /dev/null @@ -1,604 +0,0 @@ -#!/usr/bin/perl -# This script was originally based on the script of the same name from -# the KDE SDK (by dfaure@kde.org) -# -# This version is -# Copyright (C) 2007, 2008 Adam D. Barratt -# Copyright (C) 2012 Francesco Poli -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - -=head1 NAME - -licensecheck - simple license checker for source files - -=head1 SYNOPSIS - -B B<--help>|B<--version> - -B [B<--no-conf>] [B<--verbose>] [B<--copyright>] -[B<-l>|B<--lines=>I] [B<-i>|B<--ignore=>I] [B<-c>|B<--check=>I] -[B<-m>|B<--machine>] [B<-r>|B<--recursive>] -I - -=head1 DESCRIPTION - -B attempts to determine the license that applies to each file -passed to it, by searching the start of the file for text belonging to -various licenses. - -If any of the arguments passed are directories, B will add -the files contained within to the list of files to process. - -=head1 OPTIONS - -=over 4 - -=item B<--verbose>, B<--no-verbose> - -Specify whether to output the text being processed from each file before -the corresponding license information. - -Default is to be quiet. - -=item B<-l=>I, B<--lines=>I - -Specify the number of lines of each file's header which should be parsed -for license information. (Default is 60). - -=item B<-i=>I, B<--ignore=>I - -When processing the list of files and directories, the regular -expression specified by this option will be used to indicate those which -should not be considered (e.g. backup files, VCS metadata). - -=item B<-r>, B<--recursive> - -Specify that the contents of directories should be added -recursively. - -=item B<-c=>I, B<--check=>I - -Specify a pattern against which filenames will be matched in order to -decide which files to check the license of. - -The default includes common source files. - -=item B<--copyright> - -Also display copyright text found within the file - -=item B<-m>, B<--machine> - -Display the information in a machine readable way, i.e. in the form -[] so that it can be easily sorted -and/or filtered, e.g. with the B and B commands. -Note that using the B<--verbose> option will kill the readability. - -=item B<--no-conf>, B<--noconf> - -Do not read any configuration files. This can only be used as the first -option given on the command line. - -=back - -=head1 CONFIGURATION VARIABLES - -The two configuration files F and -F<~/.devscripts> are sourced by a shell in that order to set -configuration variables. Command line options can be used to override -configuration file settings. Environment variable settings are -ignored for this purpose. The currently recognised variables are: - -=over 4 - -=item B - -If this is set to I, then it is the same as the B<--verbose> command -line parameter being used. The default is I. - -=item B - -If this is set to a positive number then the specified number of lines -at the start of each file will be read whilst attempting to determine -the license(s) in use. This is equivalent to the B<--lines> command line -option. - -=back - -=head1 LICENSE - -This code is copyright by Adam D. Barratt >, -all rights reserved; based on a script of the same name from the KDE -SDK, which is copyright by >. -This program comes with ABSOLUTELY NO WARRANTY. -You are free to redistribute this code under the terms of the GNU -General Public License, version 2 or later. - -=head1 AUTHOR - -Adam D. Barratt - -=cut - -use strict; -use warnings; -use Getopt::Long qw(:config gnu_getopt); -use File::Basename; - -my $progname = basename($0); - -# From dpkg-source -my $default_ignore_regex = ' -# Ignore general backup files -(?:^|/).*~$| -# Ignore emacs recovery files -(?:^|/)\.#.*$| -# Ignore vi swap files -(?:^|/)\..*\.swp$| -# Ignore baz-style junk files or directories -(?:^|/),,.*(?:$|/.*$)| -# File-names that should be ignored (never directories) -(?:^|/)(?:DEADJOE|\.cvsignore|\.arch-inventory|\.bzrignore|\.gitignore)$| -# File or directory names that should be ignored -(?:^|/)(?:CVS|RCS|\.pc|\.deps|\{arch\}|\.arch-ids|\.svn|\.hg|_darcs|\.git| -\.shelf|_MTN|\.bzr(?:\.backup|tags)?)(?:$|/.*$) -'; - -# Take out comments and newlines -$default_ignore_regex =~ s/^#.*$//mg; -$default_ignore_regex =~ s/\n//sg; - -my $default_check_regex = '\.(c(c|pp|xx)?|h(h|pp|xx)?|f(77|90)?|go|p(l|m)|xs|sh|php|py(|x)|rb|java|js|vala|el|sc(i|e)|cs|pas|inc|dtd|xsl|mod|m|tex|mli?|(c|l)?hs)$'; - -my $modified_conf_msg; - -my %OPT=( - verbose => '', - lines => '', - noconf => '', - ignore => '', - check => '', - recursive => 0, - copyright => 0, - machine => 0, -); - -my $def_lines = 60; - -# Read configuration files and then command line -# This is boilerplate - -if (@ARGV and $ARGV[0] =~ /^--no-?conf$/) { - $modified_conf_msg = " (no configuration files read)"; - shift; -} else { - my @config_files = ('/etc/devscripts.conf', '~/.devscripts'); - my %config_vars = ( - 'LICENSECHECK_VERBOSE' => 'no', - 'LICENSECHECK_PARSELINES' => $def_lines, - ); - my %config_default = %config_vars; - - my $shell_cmd; - # Set defaults - foreach my $var (keys %config_vars) { - $shell_cmd .= qq[$var="$config_vars{$var}";\n]; - } - $shell_cmd .= 'for file in ' . join(" ", @config_files) . "; do\n"; - $shell_cmd .= '[ -f $file ] && . $file; done;' . "\n"; - # Read back values - foreach my $var (keys %config_vars) { $shell_cmd .= "echo \$$var;\n" } - my $shell_out = `/bin/bash -c '$shell_cmd'`; - @config_vars{keys %config_vars} = split /\n/, $shell_out, -1; - - # Check validity - $config_vars{'LICENSECHECK_VERBOSE'} =~ /^(yes|no)$/ - or $config_vars{'LICENSECHECK_VERBOSE'} = 'no'; - $config_vars{'LICENSECHECK_PARSELINES'} =~ /^[1-9][0-9]*$/ - or $config_vars{'LICENSECHECK_PARSELINES'} = $def_lines; - - foreach my $var (sort keys %config_vars) { - if ($config_vars{$var} ne $config_default{$var}) { - $modified_conf_msg .= " $var=$config_vars{$var}\n"; - } - } - $modified_conf_msg ||= " (none)\n"; - chomp $modified_conf_msg; - - $OPT{'verbose'} = $config_vars{'LICENSECHECK_VERBOSE'} eq 'yes' ? 1 : 0; - $OPT{'lines'} = $config_vars{'LICENSECHECK_PARSELINES'}; -} - -GetOptions(\%OPT, - "help|h", - "check|c=s", - "copyright", - "ignore|i=s", - "lines|l=i", - "machine|m", - "noconf|no-conf", - "recursive|r", - "verbose!", - "version|v", -) or die "Usage: $progname [options] filelist\nRun $progname --help for more details\n"; - -$OPT{'lines'} = $def_lines if $OPT{'lines'} !~ /^[1-9][0-9]*$/; -$OPT{'ignore'} = $default_ignore_regex if ! length $OPT{'ignore'}; -$OPT{'check'} = $default_check_regex if ! length $OPT{'check'}; - -if ($OPT{'noconf'}) { - fatal("--no-conf is only acceptable as the first command-line option!"); -} -if ($OPT{'help'}) { help(); exit 0; } -if ($OPT{'version'}) { version(); exit 0; } - -die "Usage: $progname [options] filelist\nRun $progname --help for more details\n" unless @ARGV; - -$OPT{'lines'} = $def_lines if not defined $OPT{'lines'}; - -my @files = (); -my @find_args = (); -my $files_count = @ARGV; - -push @find_args, qw(-maxdepth 1) unless $OPT{'recursive'}; -push @find_args, qw(-follow -type f -print); - -while (@ARGV) { - my $file = shift @ARGV; - - if (-d $file) { - open my $FIND, '-|', 'find', $file, @find_args - or die "$progname: couldn't exec find: $!\n"; - - while (<$FIND>) { - chomp; - next unless m%$OPT{'check'}%; - # Skip empty files - next if (-z $_); - push @files, $_ unless m%$OPT{'ignore'}%; - } - close $FIND; - } else { - next unless ($files_count == 1) or $file =~ m%$OPT{'check'}%; - push @files, $file unless $file =~ m%$OPT{'ignore'}%; - } -} - -while (@files) { - my $file = shift @files; - my $content = ''; - my $copyright_match; - my $copyright = ''; - my $license = ''; - my %copyrights; - - open (my $F, '<' ,$file) or die "Unable to access $file\n"; - while (<$F>) { - last if ($. > $OPT{'lines'}); - $content .= $_; - $copyright_match = parse_copyright($_); - if ($copyright_match) { - $copyrights{lc("$copyright_match")} = "$copyright_match"; - } - } - close($F); - - $copyright = join(" / ", reverse sort values %copyrights); - - print qq(----- $file header -----\n$content----- end header -----\n\n) - if $OPT{'verbose'}; - - $license = parselicense(clean_comments($content)); - - if ($OPT{'machine'}) { - print "$file\t$license"; - print "\t" . ($copyright or "*No copyright*") if $OPT{'copyright'}; - print "\n"; - } else { - print "$file: "; - print "*No copyright* " unless $copyright; - print $license . "\n"; - print " [Copyright: " . $copyright . "]\n" - if $copyright and $OPT{'copyright'}; - print "\n" if $OPT{'copyright'}; - } -} - -sub parse_copyright { - my $copyright = ''; - my $match; - - my $copyright_indicator_regex = ' - (?:copyright # The full word - |copr\. # Legally-valid abbreviation - |\x{00a9} # Unicode character COPYRIGHT SIGN - |\xc2\xa9 # Unicode copyright sign encoded in iso8859 - |\(c\) # Legally-null representation of sign - )'; - my $copyright_disindicator_regex = ' - \b(?:info(?:rmation)? # Discussing copyright information - |(notice|statement|claim|string)s? # Discussing the notice - |and|or|is|in|to # Part of a sentence - |(holder|owner)s? # Part of a sentence - |ownership # Part of a sentence - )\b'; - my $copyright_predisindicator_regex = '( - ^[#]define\s+.*\(c\) # #define foo(c) -- not copyright - )'; - - if ( ! m%$copyright_predisindicator_regex%ix) { - - if (m%$copyright_indicator_regex(?::\s*|\s+)(\S.*)$%ix) { - $match = $1; - - # Ignore lines matching "see foo for copyright information" etc. - if ($match !~ m%^\s*$copyright_disindicator_regex%ix) { - # De-cruft - $match =~ s/([,.])?\s*$//; - $match =~ s/$copyright_indicator_regex//igx; - $match =~ s/^\s+//; - $match =~ s/\s{2,}/ /g; - $match =~ s/\\@/@/g; - $copyright = $match; - } - } - } - - return $copyright; -} - -sub clean_comments { - local $_ = shift or return q{}; - - # Remove generic comments: look for 4 or more lines beginning with - # regular comment pattern and trim it. Fall back to old algorithm - # if no such pattern found. - my @matches = m/^\s*([^a-zA-Z0-9\s]{1,3})\s\w/mg; - if (@matches >= 4) { - my $comment_re = qr/\s*[\Q$matches[0]\E]{1,3}\s*/; - s/^$comment_re//mg; - } - - # Remove Fortran comments - s/^[cC] //gm; - tr/\t\r\n/ /; - - # Remove C / C++ comments - s#(\*/|/[/*])##g; - tr% A-Za-z.,@;0-9\(\)/-%%cd; - tr/ //s; - - return $_; -} - -sub help { - print <<"EOF"; -Usage: $progname [options] filename [filename ...] -Valid options are: - --help, -h Display this message - --version, -v Display version and copyright info - --no-conf, --noconf Don't read devscripts config files; must be - the first option given - --verbose Display the header of each file before its - license information - --lines, -l Specify how many lines of the file header - should be parsed for license information - (Default: $def_lines) - --check, -c Specify a pattern indicating which files should - be checked - (Default: '$default_check_regex') - --machine, -m Display in a machine readable way (good for awk) - --recursive, -r Add the contents of directories recursively - --copyright Also display the file's copyright - --ignore, -i Specify that files / directories matching the - regular expression should be ignored when - checking files - (Default: '$default_ignore_regex') - -Default settings modified by devscripts configuration files: -$modified_conf_msg -EOF -} - -sub version { - print <<"EOF"; -This is $progname, from the Debian devscripts package, version ###VERSION### -Copyright (C) 2007, 2008 by Adam D. Barratt ; based -on a script of the same name from the KDE SDK by . - -This program comes with ABSOLUTELY NO WARRANTY. -You are free to redistribute this code under the terms of the -GNU General Public License, version 2, or (at your option) any -later version. -EOF -} - -sub parselicense { - my ($licensetext) = @_; - - my $gplver = ""; - my $extrainfo = ""; - my $license = ""; - - if ($licensetext =~ /version ([^, ]+?)[.,]? (?:\(?only\)?.? )?(?:of the GNU (Affero )?(Lesser |Library )?General Public License )?(as )?published by the Free Software Foundation/i or - $licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License (?:as )?published by the Free Software Foundation[;,] version ([^, ]+?)[.,]? /i) { - - $gplver = " (v$1)"; - } elsif ($licensetext =~ /GNU (?:Affero )?(?:Lesser |Library )?General Public License, version (\d+(?:\.\d+)?)[ \.]/) { - $gplver = " (v$1)"; - } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) any later version/) { - $gplver = " (v$1 or later)"; - } elsif ($licensetext =~ /either version ([^ ]+)(?: of the License)?, or \(at your option\) version (\d(?:[\.-]\d+)*)/) { - $gplver = " (v$1 or v$2)"; - } - - if ($licensetext =~ /(?:675 Mass Ave|59 Temple Place|51 Franklin Steet|02139|02111-1307)/i) { - $extrainfo = " (with incorrect FSF address)$extrainfo"; - } - - if ($licensetext =~ /permission (?:is (also granted|given))? to link (the code of )?this program with (any edition of )?(Qt|the Qt library)/i) { - $extrainfo = " (with Qt exception)$extrainfo" - } - - if ($licensetext =~ /(All changes made in this file will be lost|DO NOT (EDIT|delete this file)|Generated (automatically|by|from)|generated.*file)/i) { - $license = "GENERATED FILE"; - } - - if ($licensetext =~ /((is free software.? )?you can redistribute (it|them) and\/or modify (it|them)|is licensed) under the terms of (version [^ ]+ of )?the (GNU (Library |Lesser )General Public License|LGPL)/i) { - $license = "LGPL$gplver$extrainfo $license"; - } - - if ($licensetext =~ /is free software.? you can redistribute (it|them) and\/or modify (it|them) under the terms of the (GNU Affero General Public License|AGPL)/i) { - $license = "AGPL$gplver$extrainfo $license"; - } - - if ($licensetext =~ /(is free software.? )?you (can|may) redistribute (it|them) and\/or modify (it|them) under the terms of (?:version [^ ]+ (?:\(?only\)? )?of )?the GNU General Public License/i) { - $license = "GPL$gplver$extrainfo $license"; - } - - if ($licensetext =~ /is distributed under the terms of the GNU General Public License,/ - and length $gplver) { - $license = "GPL$gplver$extrainfo $license"; - } - - if ($licensetext =~ /is distributed.*terms.*GPL/) { - $license = "GPL (unversioned/unknown version) $license"; - } - - if ($licensetext =~ /This file is part of the .*Qt GUI Toolkit. This file may be distributed under the terms of the Q Public License as defined/) { - $license = "QPL (part of Qt) $license"; - } elsif ($licensetext =~ /may (be distributed|redistribute it) under the terms of the Q Public License/) { - $license = "QPL $license"; - } - - if ($licensetext =~ /opensource\.org\/licenses\/mit-license\.php/) { - $license = "MIT/X11 (BSD like) $license"; - } elsif ($licensetext =~ /Permission is hereby granted, free of charge, to any person obtaining a copy of this software and(\/or)? associated documentation files \(the (Software|Materials)\), to deal in the (Software|Materials)/) { - $license = "MIT/X11 (BSD like) $license"; - } elsif ($licensetext =~ /Permission is hereby granted, without written agreement and without license or royalty fees, to use, copy, modify, and distribute this software and its documentation for any purpose/) { - $license = "MIT/X11 (BSD like) $license"; - } - - if ($licensetext =~ /Permission to use, copy, modify, and(\/or)? distribute this software for any purpose with or without fee is hereby granted, provided.*copyright notice.*permission notice.*all copies/) { - $license = "ISC $license"; - } - - if ($licensetext =~ /THIS SOFTWARE IS PROVIDED .*AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY/) { - if ($licensetext =~ /All advertising materials mentioning features or use of this software must display the following acknowledge?ment.*This product includes software developed by/i) { - $license = "BSD (4 clause) $license"; - } elsif ($licensetext =~ /(The name(?:\(s\))? .*? may not|Neither the (names? .*?|authors?) nor the names of( (its|their|other|any))? contributors may) be used to endorse or promote products derived from this software/i) { - $license = "BSD (3 clause) $license"; - } elsif ($licensetext =~ /Redistributions of source code must retain the above copyright notice/i) { - $license = "BSD (2 clause) $license"; - } else { - $license = "BSD $license"; - } - } - - if ($licensetext =~ /Mozilla Public License,? (Version|v\.) (\d+(?:\.\d+)?)/) { - $license = "MPL (v$2) $license"; - } - - if ($licensetext =~ /Released under the terms of the Artistic License ([^ ]+)/) { - $license = "Artistic (v$1) $license"; - } - - if ($licensetext =~ /is free software under the Artistic [Ll]icense/) { - $license = "Artistic $license"; - } - - if ($licensetext =~ /This program is free software; you can redistribute it and\/or modify it under the same terms as Perl itself/) { - $license = "Perl $license"; - } - - if ($licensetext =~ /under the Apache License, Version ([^ ]+)/) { - $license = "Apache (v$1) $license"; - } - - if ($licensetext =~ /(THE BEER-WARE LICENSE)/i) { - $license = "Beerware $license"; - } - - if ($licensetext =~ /This source file is subject to version ([^ ]+) of the PHP license/) { - $license = "PHP (v$1) $license"; - } - - if ($licensetext =~ /under the terms of the CeCILL /) { - $license = "CeCILL $license"; - } - - if ($licensetext =~ /under the terms of the CeCILL-([^ ]+) /) { - $license = "CeCILL-$1 $license"; - } - - if ($licensetext =~ /under the SGI Free Software License B/) { - $license = "SGI Free Software License B $license"; - } - - if ($licensetext =~ /is in the public domain/i) { - $license = "Public domain $license"; - } - - if ($licensetext =~ /terms of the Common Development and Distribution License(, Version ([^(]+))? \(the License\)/) { - $license = "CDDL " . ($1 ? "(v$2) " : '') . $license; - } - - if ($licensetext =~ /Microsoft Permissive License \(Ms-PL\)/) { - $license = "Ms-PL $license"; - } - - if ($licensetext =~ /Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by this license \(the \"Software\"\)/ or - $licensetext =~ /Boost Software License([ ,-]+Version ([^ ]+)?(\.))/i) { - $license = "BSL " . ($1 ? "(v$2) " : '') . $license; - } - - if ($licensetext =~ /PYTHON SOFTWARE FOUNDATION LICENSE (VERSION ([^ ]+))/i) { - $license = "PSF " . ($1 ? "(v$2) " : '') . $license; - } - - if ($licensetext =~ /The origin of this software must not be misrepresented.*Altered source versions must be plainly marked as such.*This notice may not be removed or altered from any source distribution/ or - $licensetext =~ /see copyright notice in zlib\.h/) { - $license = "zlib/libpng $license"; - } elsif ($licensetext =~ /This code is released under the libpng license/) { - $license = "libpng $license"; - } - - if ($licensetext =~ /Do What The Fuck You Want To Public License, Version ([^, ]+)/i) { - $license = "WTFPL (v$1) $license"; - } - - if ($licensetext =~ /Do what The Fuck You Want To Public License/i) { - $license = "WTFPL $license"; - } - - if ($licensetext =~ /(License WTFPL|Under (the|a) WTFPL)/i) { - $license = "WTFPL $license"; - } - - $license = "UNKNOWN" if (!length($license)); - - # Remove trailing spaces. - $license =~ s/\s+$//; - - return $license; -} - -sub fatal { - my ($pack,$file,$line); - ($pack,$file,$line) = caller(); - (my $msg = "$progname: fatal error at line $line:\n@_\n") =~ tr/\0//d; - $msg =~ s/\n\n$/\n/; - die $msg; -} diff --git a/scripts/licensecheck.py b/scripts/licensecheck.py deleted file mode 100755 index 08d0e1b4..00000000 --- a/scripts/licensecheck.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import argparse -import errno -import os -import pipes -import re -import string -import subprocess -import sys -import tempfile - -import scriptslib - - -gpl3_chunks = ("autoconf", - "autoconf-tarball", - "automake", - "bash", - "binutils", - "bison", - "ccache", - "cmake", - "flex", - "gawk", - "gcc", - "gdbm", - "gettext-tarball", - "gperf", - "groff", - "libtool", - "libtool-tarball", - "m4-tarball", - "make", - "nano", - "patch", - "rsync", - "texinfo-tarball") - - -def license_file_name(repo_name, sha, licenses_dir): - license_file = os.path.join(licenses_dir, repo_name + '-' + sha) - return license_file - - -def check_license(repo_name, sha, clone_path, license_file): - - licenses_dir = os.path.dirname(license_file) - - try: - os.makedirs(licenses_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - _, license_file_temp = tempfile.mkstemp(dir=licenses_dir) - with open(license_file_temp,"wb") as out: - sys.stderr.write("Checking license of '%s' ...\n" % repo_name) - with open(os.devnull, 'w') as devnull: - subprocess.check_call("perl scripts/licensecheck.pl -r " - + pipes.quote(clone_path) + "|cut -d: -f2- | sort -u", - stdout=out, stderr=devnull, shell=True) - - os.rename(license_file_temp, license_file) - return license_file - - -def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir): - repo_name = re.split('/|:',repo)[-1] - if repo_name.endswith(".git"): - repo_name = repo_name[:-4] - - repo_url = scriptslib.parse_repo_alias(repo) - - # Check if ref is sha1 to speedup - if len(ref) == 40 and all(c in string.hexdigits for c in ref): - license_file = license_file_name(repo_name, ref, licenses_dir) - if os.path.isfile(license_file): - return (repo, license_file) - - clone_path = os.path.join(repos_dir, repo_name) - - if os.path.isdir(clone_path): - sys.stderr.write("Updating repo '%s' ...\n" % repo_name) - with open(os.devnull, 'w') as devnull: - subprocess.check_call([ - "git", "remote", "update", "origin", "--prune"], - stderr=devnull, stdout=devnull, cwd=clone_path) - # Update submodules - subprocess.check_call( - ["git", "submodule", "update", "--recursive"], - stderr=devnull, stdout=devnull, cwd=clone_path) - subprocess.check_call(["git", "checkout", ref], stderr=devnull, - stdout=devnull, cwd=clone_path) - else: - sys.stderr.write("Getting repo '%s' ...\n" % repo_name) - with open(os.devnull, 'w') as devnull: - try: - # Attempt to use morph to obtain a repository, from morph's - # existing local git cache if possible - subprocess.check_call( - ["morph", "get-repo", name, clone_path], - stdout=devnull, stderr=devnull) - - except (OSError, subprocess.CalledProcessError): - # Fall back to git clone, when morph hasn't been found on the - # system, or otherwise fails to get a repo. This is required - # where morph isn't available, e.g. when using YBD to build. - # YBD currently doesn't offer a similar 'get-repo' feature. - sys.stderr.write("Falling back to git clone.\n") - subprocess.check_call( - ["git", "clone", "--recursive", repo_url, clone_path], - stdout=devnull, stderr=devnull) # also clone submodules - - sha = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=clone_path).strip() - - license_file = license_file_name(repo_name, sha, licenses_dir) - if os.path.isfile(license_file): - return (repo, license_file) - - return (repo, check_license(repo_name, sha, clone_path, license_file)) - - -def check_stratum(stratum_file, repos_dir, licenses_dir): - stratum = scriptslib.load_yaml_file(stratum_file) - license_files = [] - for chunk in stratum['chunks']: - - name = chunk["name"] - build_mode = chunk.get("build-mode") # Allowed to be None - - # Don't include bootstrap chunks and stripped gplv3 chunks - if name in gpl3_chunks or build_mode == "bootstrap": - continue - repo = chunk["repo"] - ref = chunk["ref"] - yield check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir) - - -def main(): - - parser = argparse.ArgumentParser( - description='Checks licenses of the components of a given System.') - parser.add_argument('system', metavar='SYSTEM', type=str, - help='System to check for licenses') - parser.add_argument('--repos-dir', default="./repos", - help='DIR to clone all the repos (default ./repos)') - parser.add_argument('--licenses-dir', default="./licenses", - help='DIR to store chunk license files (default ./licenses)') - - args = parser.parse_args() - - if not os.path.exists(args.repos_dir): - os.makedirs(args.repos_dir) - - system = scriptslib.load_yaml_file(args.system) - license_files = [] - for stratum in system['strata']: - stratum_file = stratum['morph'] - stratum_path = os.path.join(scriptslib.definitions_root(), stratum_file) - license_files.extend(check_stratum(stratum_path, args.repos_dir, args.licenses_dir)) - - for chunk_repo, chunk_license in license_files: - try: - # Print repo name - sys.stdout.write("%s\n%s\n" % (chunk_repo, '-' * len(chunk_repo))) - - # Print license file of the repo - with open(chunk_license, 'r') as f: - for line in f: - sys.stdout.write(line) - sys.stdout.write("\n") - except IOError: - # stdout is closed, no point in continuing - # Attempt to close them explicitly to prevent cleanup problems: - try: - sys.stdout.flush() - sys.stdout.close() - except IOError: - pass - finally: - exit() - - -if __name__ == "__main__": - main() diff --git a/scripts/organize-morphologies.py b/scripts/organize-morphologies.py deleted file mode 100755 index 3072c8f8..00000000 --- a/scripts/organize-morphologies.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2014-2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import json -import morphlib -import os -import subprocess -import sys -import urllib -import urllib2 -import urlparse -import yaml -import re -import errno - -''' organize-morphologies.py: -Tool for organizing morphologies in definitions. - -This script will move: - - cluster morphologies into clusters directory - - system morphologies into systems directory - - stratum morphologies into strata directory - -This script will download the chunk morphologies for every stratum -and placed into strata/stratum_which_the_chunk_belongs_to directory. - -It also modifies the morphologies fields which points to some morpholgy -which has been moved. -''' - - -def make_request(path): - server_url = 'http://git.baserock.org:8080/' - url = urlparse.urljoin(server_url, '/1.0/%s' % path) - handle = urllib2.urlopen(url) - return handle.read() - -def quote(*args): - return tuple(urllib.quote(string) for string in args) - -def cat_file(repo, ref, filename): - return make_request('files?repo=%s&ref=%s&filename=%s' % - quote(repo, ref, filename)) - -# NOTE: This function reimplement part of morphlib's loader -def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'): - '''This function receives the name or the morph field of one morphology - and returns the path of the morphology depending on the name, kind and - if it belongs to other morphologies. - ''' - # Dictionary which match morphology's kind and morphology's - # directory in definitions.git - morph_dir = { 'chunk': 'chunks', 'stratum': 'strata', - 'system':'systems', 'cluster': 'clusters'} - # For chunks morphologies we need to know to which stratums - # belongs this chunk. - if morph_kind == 'chunk': - if belongs_to == '': - raise morphlib.Error('Chunk morphologies need the stratum name' - 'to create the path. Please add the stratum' - 'which belongs this morphology') - # Get the name of the chunk which we assume is at the end - # of the morph file - if '/' in morph_field: - morph_field = os.path.basename(morph_field) - - # Add the stratum name to the chunk name - morph_field = os.path.join(belongs_to, morph_field) - - # Reset the kind to stratum because chunk contains stratum - # name in its path. - morph_kind = 'stratum' - - # Add the morphology path to the morph field. - if not morph_field.startswith(morph_dir[morph_kind]): - morph_field = os.path.join(morph_dir[morph_kind], morph_field) - - # Add the morphology suffix if the morphology. - if not morph_field.endswith('.morph'): - morph_field = morph_field + '.morph' - - return morph_field - -def create_directory(name, path): - directory = os.path.join(path, name) - try: - os.makedirs(directory) - except OSError as err: - if err.errno != errno.EEXIST: - raise err - else: - pass - return directory - -def move_file(morph, directory, path, loader): - if not morph.filename.startswith(directory): - filename = os.path.basename(morph.filename) - new_location = os.path.join(path, filename) - print '\nMoving %s into %s' % (filename, new_location) - subprocess.call(['git', 'mv', morph.filename, new_location]) - morph.filename = new_location - loader.unset_defaults(morph) - loader.save_to_file(morph.filename, morph) - -def load_and_fix_chunk(chunk_str, loader, name): - try: - chunk_morph = loader.load_from_string(chunk_str) - except morphlib.morphloader.InvalidFieldError as err: - if "comments" in str(err): - # This error is caused because there are old morphologies which - # contain the field "comments" instead of "description". - # Replacing "comments" field by "description" will allow the morphology - # to pass parse_morphology_text check and ready to be written to a file. - fixed_chunk = loader.parse_morphology_text(chunk_str, name) - fixed_chunk['description'] = fixed_chunk.pop('comments') - print "WARNING: Invalid 'comments' field in " \ - "%s corrected to 'description'" % name - chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name) - elif "buildsystem" in str(err): - # This error is caused because a typo in a morphology which - # has a field "buildsystem" instead of "build-system". - fixed_chunk = loader.parse_morphology_text(chunk_str, name) - fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem') - print "WARNING: Invalid 'buildsystem' field in %s" \ - "corrected to 'build-system'" % name - chunk_morph = load_and_fix_chunk(str(fixed_chunk), loader, name) - else: - print "ERROR: %s in chunk %s" %(err, name) - raise err - except morphlib.morphloader.MorphologyNotYamlError as err: - print "WARNING: %s in chunk %s is not valid YAML, " \ - "attempting to fix..." %(err, name) - # This error is caused because there are old morphologies written - # in JSON which contain '\t' characters. When try to load this - # kind of morphologies load_from_string fails when parse_morphology_text. - # Removing this characters will make load_from_string to load the morphology - # and translate it into a correct yaml format. - fixed_chunk = chunk_str.replace('\t','') - print "INFO: %s successfully fixed" % name - chunk_morph = load_and_fix_chunk(fixed_chunk, loader, name) - return chunk_morph - -def move_clusters(morphs, path, loader): - kind = 'system' - directory = 'clusters' - # Move cluster morphologies to clusters folder fixing their dependent - # morphologies which are systems. - full_path = create_directory(directory, path) - for morph in morphs: - all_systems = morph['systems'][:] - for system in morph['systems']: - all_systems.extend(system.get('subsystems', [])) - # Add the correct path to the morph fields for systems and subsystems - for field in all_systems: - field['morph'] = sanitise_morphology_path(field['morph'], kind) - move_file(morph, directory, full_path, loader) - -def move_systems(morphs, path, loader): - kind = 'stratum' - directory = 'systems' - # Move system morphologies to systems folder fixing their dependent - # morphologies which are strata. - full_path = create_directory(directory, path) - for morph in morphs: - # Add name field and the correct path to the stratum on the morph - # fields in strata. - for field in morph['strata']: - field['name'] = os.path.basename(field['morph']) - field['morph'] = sanitise_morphology_path(field['morph'], kind) - move_file(morph, directory, full_path, loader) - -def download_chunks(morph, loader): - # Download chunks morphologies defined on the stratum and - # add them to the directory tree. - for chunk in morph['chunks']: - name = chunk['name'] + '.morph' - try: - chunk['morph'] = sanitise_morphology_path(chunk['morph'], 'chunk', morph['name']) - except KeyError as err: - if 'morph' in str(err): - chunk['morph'] = sanitise_morphology_path(chunk['name'], 'chunk', morph['name']) - else: - raise err - ref = chunk['ref'] - repo = scriptslib.parse_repo_alias(chunk['repo']) - try: - print "\nDownloading %s from %s into %s" %(name, repo, chunk['morph']) - chunk_str = cat_file(repo, ref, name) - except urllib2.HTTPError as err: - # If there is no morphology in the repository we assume that the morphology - # system will be autodetected, so we don't have to create a new one - # unless we shut down the autodetecting system (fallback system). - if err.code == 404: - print 'INFO: Morph will fall-back to build-time' \ - 'autodetection for %s' %(name) - # Remove morph field from autodetected chunks - del chunk['morph'] - else: - loaded_chunk = load_and_fix_chunk(chunk_str, loader, name) - loader.unset_defaults(loaded_chunk) - loader.save_to_file(chunk['morph'], loaded_chunk) - -def move_strata(morphs, path, loader): - # Create strata directory - strata_dir = 'strata/' - strata_path = create_directory(strata_dir, path) - for morph in morphs: - # Create stratum directory where downloading its chunks. - stratum_path = strata_path + morph['name'] - stratum_dir = create_directory(stratum_path, path) - - # Download chunks which belongs to the stratum - download_chunks(morph, loader) - - # Add to build-depends the correct path to the dependent stratum morphologies. - for build_depends in morph['build-depends']: - build_depends['morph'] = sanitise_morphology_path(build_depends['morph'], 'stratum') - # Move stratum morphologies to strata - move_file(morph, strata_dir, strata_path, loader) - -def main(): - # Load all morphologies in the definitions repo - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - morphs = [m for m in sb.load_all_morphologies(loader)] - - # Clasify the morphologies regarding of their kind field - morphologies = { kind: [m for m in morphs if m['kind'] == kind] - for kind in ('chunk', 'stratum', 'system', 'cluster') } - - for kind, morphs in morphologies.iteritems(): - print 'There are: %d %s.\n' %(len(morphs), kind) - - # Get the path from definitions repo - definitions_repo = sb.get_git_directory_name(sb.root_repository_url) - - # Move the morphologies to its directories - move_clusters(morphologies['cluster'], definitions_repo, loader) - move_systems(morphologies['system'], definitions_repo, loader) - move_strata(morphologies['stratum'], definitions_repo, loader) - -main() diff --git a/scripts/release-build b/scripts/release-build deleted file mode 100755 index cb62f661..00000000 --- a/scripts/release-build +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import morphlib -import os -import subprocess -import sys -import time - - -class Build(object): - '''A single distbuild instance.''' - - def __init__(self, name, arch, app): - self.system_name = name - - controller_netloc = app.controllers[arch].split(':') - controller_args = [ - '--controller-initiator-address=%s' % controller_netloc[0], - ] - if len(controller_netloc) > 1: - controller_args.append( - '--controller-initiator-port=%s' % controller_netloc[1]) - - self.command = ['morph', 'distbuild', '--local-changes=ignore'] - self.command += controller_args + [self.system_name] - - def start(self): - self.process = subprocess.Popen(self.command) - - def completed(self): - return (self.process.poll() is not None) - - -class ReleaseApp(cliapp.Application): - - '''Cliapp app that handles distbuilding and deploying a cluster.''' - - def add_settings(self): - self.settings.string_list(['controllers'], - 'a list of distbuild controllers and their ' - 'architecture') - - self.settings.string(['trove-host'], - 'hostname of Trove instance') - - self.settings.string(['artifact-cache-server'], - 'server to fetch artifacts from', default=None) - - self.settings.string(['release-number'], - 'Baserock version of the systems being built', - default='yy.ww') - - def error(self, message): - raise cliapp.AppException(message) - - def check_args(self, args): - if len(args) == 0: - self.error( - "Please pass the name of the release cluster (e.g. " - "clusters/release.morph)") - - if len(args) > 1: - self.error("Too many arguments given.") - - def process_args(self, args): - '''Process the command line''' - self.controllers = {} - controllers_list = self.settings['controllers'] - - for item in controllers_list: - arch, controller = item.split(':', 1) - self.controllers[arch] = controller - - defs_repo = morphlib.definitions_repo.open( - '.', search_for_root=True) - self.loader = defs_repo.get_morphology_loader() - self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) - - self.check_args(args) - - cluster_name = args[0] - cluster, cluster_path = self.load_morphology(cluster_name) - - builds = self.prepare_builds(cluster) - for build in builds: - build.start() - - while not all(build.completed() for build in builds): - time.sleep(1) - - fail = False - for build in builds: - if build.process.returncode != 0: - fail = True - sys.stderr.write( - 'Building failed for %s\n' % build.system_name) - if fail: - raise cliapp.AppException('Building of systems failed') - - if not os.path.exists('release'): - os.mkdir('release') - self.deploy_images(cluster, cluster_path) - - def load_morphology(self, name, kind=None): - path = morphlib.util.sanitise_morphology_path(name) - morph = self.loader.load_from_string( - self.finder.read_file(path)) - if kind: - assert morph['kind'] == kind - return morph, path - - def iterate_systems(self, system_list): - for system in system_list: - yield system['morph'] - if 'subsystems' in system: - for subsystem in self.iterate_systems(system['subsystems']): - yield subsystem - - def prepare_builds(self, cluster): - '''Prepare a list of builds''' - systems = set(self.iterate_systems(cluster['systems'])) - builds = [] - for system_name in systems: - system, _ = self.load_morphology(system_name) - if system['arch'] in self.controllers: - builds.append(Build(system_name, system['arch'], self)) - else: - print("Unable to build %s: no %s distbuild available" % - (system_name, system['arch'])) - return builds - - def deploy_images(self, cluster, cluster_path): - version_label = 'baserock-%s' % self.settings['release-number'] - outputs = {} - - for system in cluster['systems']: - morphology_name = system['morph'] - morphology = self.load_morphology(morphology_name)[0] - if morphology['arch'] not in self.controllers: - continue - - for deployment_name, deployment_info in system['deploy'].iteritems(): - # The release.morph cluster must specify a basename for the file, - # of name and extension. This script knows about name, but it - # can't find out the appropriate file extension without second - # guessing the behaviour of write extensions. - basename = deployment_info['location'] - - if '/' in basename or basename.startswith(version_label): - raise cliapp.AppException( - 'In %s: system %s.location should be just the base name, ' - 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name)) - - filename = os.path.join('release', '%s-%s' % (version_label, basename)) - if os.path.exists(filename): - self.output.write('Reusing existing deployment of %s\n' % filename) - else: - self.output.write('Creating %s from release.morph\n' % filename) - self.deploy_single_image(cluster_path, deployment_name, filename, version_label) - - def deploy_single_image(self, cluster_path, name, location, version_label): - deploy_command = [ - 'morph', 'deploy', cluster_path, name, - '--trove-host=%s' % self.settings['trove-host']] - artifact_server = self.settings['artifact-cache-server'] - if artifact_server is not None: - deploy_command.append('--artifact-cache-server=%s' % artifact_server) - deploy_command.extend(( - '%s.location=%s' % (name, location), - '%s.VERSION_LABEL=%s' % (name, version_label) - )) - - cliapp.runcmd(deploy_command, stdout=sys.stdout) - - -ReleaseApp().run() diff --git a/scripts/release-build.test.conf b/scripts/release-build.test.conf deleted file mode 100644 index 50083352..00000000 --- a/scripts/release-build.test.conf +++ /dev/null @@ -1,6 +0,0 @@ -[config] -trove-host = ct-mcr-1.ducie.codethink.co.uk -controllers = x86_64:ct-mcr-1-distbuild-x86-64-majikthise-controller.dyn.ducie.codethink.co.uk, - x86_32:ct-mcr-1-distbuild-x86-32-majikthise-controller.dyn.ducie.codethink.co.uk, - armv7lhf:ct-mcr-1-distbuild-armv7lhf-jetson.dyn.ducie.codethink.co.uk -release-number = 14.29 diff --git a/scripts/release-test b/scripts/release-test deleted file mode 100755 index 4dcc6f76..00000000 --- a/scripts/release-test +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 Codethink Ltd -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -'''release-test - -This script deploys the set of systems in the cluster morphology it is -instructed to read, to test that they work correctly. - -''' - -import cliapp -import os -import pipes -import shlex -import shutil -import socket -import tempfile -import time -import uuid - -import morphlib - - -class MorphologyHelper(object): - - def __init__(self): - self.defs_repo = morphlib.definitions_repo.open( - '.', search_for_root=True) - self.loader = morphlib.morphloader.MorphologyLoader() - self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) - - def load_morphology(self, path): - text = self.finder.read_file(path) - return self.loader.load_from_string(text) - - @classmethod - def iterate_systems(cls, systems_list): - for system in systems_list: - yield morphlib.util.sanitise_morphology_path(system['morph']) - if 'subsystems' in system: - for subsystem in cls.iterate_systems(system['subsystems']): - yield subsystem - - def iterate_cluster_deployments(cls, cluster_morph): - for system in cluster_morph['systems']: - path = morphlib.util.sanitise_morphology_path(system['morph']) - defaults = system.get('deploy-defaults', {}) - for name, options in system['deploy'].iteritems(): - config = dict(defaults) - config.update(options) - yield path, name, config - - def load_cluster_systems(self, cluster_morph): - for system_path in set(self.iterate_systems(cluster_morph['systems'])): - system_morph = self.load_morphology(system_path) - yield system_path, system_morph - - -class TimeoutError(cliapp.AppException): - - """Error to be raised when a connection waits too long""" - - def __init__(self, msg): - super(TimeoutError, self).__init__(msg) - - -class VMHost(object): - - def __init__(self, user, address, disk_path): - self.user = user - self.address = address - self.disk_path = disk_path - - @property - def ssh_host(self): - return '{user}@{address}'.format(user=self.user, address=self.address) - - def runcmd(self, *args, **kwargs): - cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) - - def virsh(self, *args, **kwargs): - self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs) - - -class DeployedSystemInstance(object): - - def __init__(self, deployment, config, host_machine, vm_id, rootfs_path): - self.deployment = deployment - self.config = config - # TODO: Stop assuming test machine can DHCP and be assigned its - # hostname in the deployer's resolve search path. - self.ip_address = self.config['HOSTNAME'] - self.host_machine = host_machine - self.vm_id = vm_id - self.rootfs_path = rootfs_path - - @property - def ssh_host(self): - # TODO: Stop assuming we ssh into test instances as root - return 'root@{host}'.format(host=self.ip_address) - - def runcmd(self, argv, chdir='.', **kwargs): - ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] - cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] - cmd += argv - ssh_cmd.append(' '.join(map(pipes.quote, cmd))) - return cliapp.runcmd(ssh_cmd, **kwargs) - - def _wait_for_dhcp(self, timeout): - '''Block until given hostname resolves successfully. - - Raises TimeoutError if the hostname has not appeared in 'timeout' - seconds. - - ''' - start_time = time.time() - while True: - try: - socket.gethostbyname(self.ip_address) - return - except socket.gaierror: - pass - if time.time() > start_time + timeout: - raise TimeoutError("Host %s did not appear after %i seconds" % - (self.ip_address, timeout)) - time.sleep(0.5) - - def _wait_for_ssh(self, timeout): - """Wait until the deployed VM is responding via SSH""" - start_time = time.time() - while True: - try: - self.runcmd(['true'], stdin=None, stdout=None, stderr=None) - return - except cliapp.AppException: - # TODO: Stop assuming the ssh part of the command is what failed - if time.time() > start_time + timeout: - raise TimeoutError("%s sshd did not start after %i seconds" - % (self.ip_address, timeout)) - time.sleep(0.5) - - def wait_until_online(self, timeout=10): - self._wait_for_dhcp(timeout) - self._wait_for_ssh(timeout) - - def delete(self): - # Stop and remove VM - try: - self.host_machine.virsh('destroy', self.vm_id) - except cliapp.AppException as e: - # TODO: Stop assuming that destroy failed because it wasn't running - pass - try: - self.host_machine.virsh('undefine', self.vm_id, '--remove-all-storage') - except cliapp.AppException as e: - # TODO: Stop assuming that undefine failed because it was - # already removed - pass - - -class Deployment(object): - - def __init__(self, cluster_path, name, deployment_config, host_machine): - self.cluster_path = cluster_path - self.name = name - self.deployment_config = deployment_config - self.host_machine = host_machine - - @staticmethod - def _ssh_host_key_exists(hostname): - """Check if an ssh host key exists in known_hosts""" - if not os.path.exists('/root/.ssh/known_hosts'): - return False - with open('/root/.ssh/known_hosts', 'r') as known_hosts: - return any(line.startswith(hostname) for line in known_hosts) - - def _update_known_hosts(self): - if not self._ssh_host_key_exists(self.host_machine.address): - with open('/root/.ssh/known_hosts', 'a') as known_hosts: - cliapp.runcmd(['ssh-keyscan', self.host_machine.address], - stdout=known_hosts) - - @staticmethod - def _generate_sshkey_config(tempdir, config): - manifest = os.path.join(tempdir, 'manifest') - with open(manifest, 'w') as f: - f.write('0040700 0 0 /root/.ssh\n') - f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') - authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') - os.makedirs(os.path.dirname(authkeys)) - with open(authkeys, 'w') as auth_f: - with open('/root/.ssh/id_rsa.pub', 'r') as key_f: - shutil.copyfileobj(key_f, auth_f) - - install_files = shlex.split(config.get('INSTALL_FILES', '')) - install_files.append(manifest) - yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) - - def deploy(self): - self._update_known_hosts() - - hostname = str(uuid.uuid4()) - vm_id = hostname - image_base = self.host_machine.disk_path - rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, - hostname=hostname) - loc = 'kvm+ssh://{ssh_host}/{id}/{path}'.format( - ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) - - options = { - 'type': 'kvm', - 'location': loc, - 'AUTOSTART': 'True', - 'HOSTNAME': hostname, - 'DISK_SIZE': '20G', - 'RAM_SIZE': '2G', - 'VERSION_LABEL': 'release-test', - } - - tempdir = tempfile.mkdtemp() - try: - options.update( - self._generate_sshkey_config(tempdir, - self.deployment_config)) - - args = ['morph', 'deploy', self.cluster_path, self.name] - for k, v in options.iteritems(): - args.append('%s.%s=%s' % (self.name, k, v)) - cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) - - config = dict(self.deployment_config) - config.update(options) - - return DeployedSystemInstance(self, config, self.host_machine, - vm_id, rootpath) - finally: - shutil.rmtree(tempdir) - - -class ReleaseApp(cliapp.Application): - - """Cliapp application which handles automatic builds and tests""" - - def add_settings(self): - """Add the command line options needed""" - group_main = 'Program Options' - self.settings.string_list(['deployment-host'], - 'ARCH:HOST:PATH that VMs can be deployed to', - default=None, - group=group_main) - self.settings.string(['trove-host'], - 'Address of Trove for test systems to build from', - default=None, - group=group_main) - self.settings.string(['trove-id'], - 'ID of Trove for test systems to build from', - default=None, - group=group_main) - self.settings.string(['build-ref-prefix'], - 'Prefix of build branches for test systems', - default=None, - group=group_main) - - @staticmethod - def _run_tests(instance, system_path, system_morph, - (trove_host, trove_id, build_ref_prefix), - morph_helper, systems): - instance.wait_until_online() - - tests = [] - def baserock_build_test(instance): - instance.runcmd(['git', 'config', '--global', 'user.name', - 'Test Instance of %s' % instance.deployment.name]) - instance.runcmd(['git', 'config', '--global', 'user.email', - 'ci-test@%s' % instance.config['HOSTNAME']]) - instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', - '/src/tmp']) - def morph_cmd(*args, **kwargs): - # TODO: decide whether to use cached artifacts or not by - # adding --artifact-cache-server= --cache-server= - argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', - '--tempdir=/src/tmp', '--log-max=100M', - '--trove-host', trove_host, '--trove-id', trove_id, - '--build-ref-prefix', build_ref_prefix] - argv.extend(args) - instance.runcmd(argv, **kwargs) - - repo = morph_helper.sb.root_repository_url - ref = morph_helper.defs_repo.HEAD - sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref) - morph_cmd('init', '/src/ws') - chdir = '/src/ws' - - morph_cmd('checkout', repo, ref, chdir=chdir) - # TODO: Add a morph subcommand that gives the path to the root repository. - repo_path = os.path.relpath( - morph_helper.sb.get_git_directory_name(repo), - morph_helper.sb.root_directory) - chdir = os.path.join(chdir, ref, repo_path) - - instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) - print 'Building test systems for {sys}'.format(sys=system_path) - for to_build_path, to_build_morph in systems.iteritems(): - if to_build_morph['arch'] == system_morph['arch']: - print 'Test building {path}'.format(path=to_build_path) - morph_cmd('build', to_build_path, chdir=chdir, - stdin=None, stdout=None, stderr=None) - print 'Finished Building test systems' - - def python_smoke_test(instance): - instance.runcmd(['python', '-c', 'print "Hello World"']) - - # TODO: Come up with a better way of determining which tests to run - if 'devel' in system_path: - tests.append(baserock_build_test) - else: - tests.append(python_smoke_test) - - for test in tests: - test(instance) - - def deploy_and_test_systems(self, cluster_path, - deployment_hosts, build_test_config): - """Run the deployments and tests""" - - version = 'release-test' - - morph_helper = MorphologyHelper() - cluster_morph = morph_helper.load_morphology(cluster_path) - systems = dict(morph_helper.load_cluster_systems(cluster_morph)) - - for system_path, deployment_name, deployment_config in \ - morph_helper.iterate_cluster_deployments(cluster_morph): - - system_morph = systems[system_path] - # We can only test systems in KVM that have a BSP - if not any('bsp' in si['morph'] for si in system_morph['strata']): - continue - - # We can only test systems in KVM that we have a host for - if system_morph['arch'] not in deployment_hosts: - continue - host_machine = deployment_hosts[system_morph['arch']] - deployment = Deployment(cluster_path, deployment_name, - deployment_config, host_machine) - - instance = deployment.deploy() - try: - self._run_tests(instance, system_path, system_morph, - build_test_config, morph_helper, systems) - finally: - instance.delete() - - def process_args(self, args): - """Process the command line args and kick off the builds/tests""" - if self.settings['build-ref-prefix'] is None: - self.settings['build-ref-prefix'] = ( - os.path.join(self.settings['trove-id'], 'builds')) - for setting in ('deployment-host', 'trove-host', - 'trove-id', 'build-ref-prefix'): - self.settings.require(setting) - - deployment_hosts = {} - for host_config in self.settings['deployment-host']: - arch, address = host_config.split(':', 1) - user, address = address.split('@', 1) - address, disk_path = address.split(':', 1) - if user == '': - user = 'root' - # TODO: Don't assume root is the user with deploy access - deployment_hosts[arch] = VMHost(user, address, disk_path) - - build_test_config = (self.settings['trove-host'], - self.settings['trove-id'], - self.settings['build-ref-prefix']) - - if len(args) != 1: - raise cliapp.AppException('Usage: release-test CLUSTER') - cluster_path = morphlib.util.sanitise_morphology_path(args[0]) - self.deploy_and_test_systems(cluster_path, deployment_hosts, - build_test_config) - - -if __name__ == '__main__': - ReleaseApp().run() diff --git a/scripts/release-test-os b/scripts/release-test-os deleted file mode 100755 index 06e01daf..00000000 --- a/scripts/release-test-os +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 Codethink Ltd -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -'''release-test - -This script deploys the set of systems in the cluster morphology it is -instructed to read, to test that they work correctly. - -''' - -import cliapp -import os -import pipes -import shlex -import shutil -import socket -import tempfile -import time -import uuid - -import morphlib - - -class NovaList: - def __init__(self): - self.output = [] - self.lines = [] - self.instance = [] - - def update(self): - self.output = cliapp.runcmd(['nova', 'list']) - self.lines = self.output.split('\n') - self.lines = self.lines[3:-2] - - def get_nova_details_for_instance(self, name): - self.update() - - for line in self.lines: - entries = line.split('|') - stripped_line = [entry.strip() for entry in entries] - if stripped_line.count(name) == 1: - self.instance = stripped_line - - def get_nova_state_for_instance(self, name): - self.get_nova_details_for_instance(name) - if not self.instance: - return - return self.instance[3] - - def get_nova_ip_for_instance(self, name): - self.get_nova_details_for_instance(name) - if not self.instance: - return - - if self.get_nova_state_for_instance(name) != 'ACTIVE': - return - - return self.instance[6] - - def get_nova_ip_for_instance_timeout(self, name, timeout=120): - start_time = time.time() - - while self.get_nova_state_for_instance(name) != 'ACTIVE': - - if time.time() > start_time + timeout: - print "%s not ACTIVE after %i seconds" % (name, timeout) - return - - time.sleep(1) - - ip_addr = self.get_nova_ip_for_instance(name) - if not ip_addr: - return - - if ip_addr.count('=') == 0: - return - - ip_addr = ip_addr[ip_addr.find('=') + 1:] - - if ip_addr.count(',') == 0: - return ip_addr - - return ip_addr[:ip_addr.find(',')] - - - -class MorphologyHelper(object): - - def __init__(self): - self.sb = sb = morphlib.sysbranchdir.open_from_within('.') - defs_repo_path = sb.get_git_directory_name(sb.root_repository_url) - self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path) - self.loader = morphlib.morphloader.MorphologyLoader() - self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) - - def load_morphology(self, path): - text = self.finder.read_file(path) - return self.loader.load_from_string(text) - - @classmethod - def iterate_systems(cls, systems_list): - for system in systems_list: - yield morphlib.util.sanitise_morphology_path(system['morph']) - if 'subsystems' in system: - for subsystem in cls.iterate_systems(system['subsystems']): - yield subsystem - - def iterate_cluster_deployments(cls, cluster_morph): - for system in cluster_morph['systems']: - path = morphlib.util.sanitise_morphology_path(system['morph']) - defaults = system.get('deploy-defaults', {}) - for name, options in system['deploy'].iteritems(): - config = dict(defaults) - config.update(options) - yield path, name, config - - def load_cluster_systems(self, cluster_morph): - for system_path in set(self.iterate_systems(cluster_morph['systems'])): - system_morph = self.load_morphology(system_path) - yield system_path, system_morph - - -class TimeoutError(cliapp.AppException): - - """Error to be raised when a connection waits too long""" - - def __init__(self, msg): - super(TimeoutError, self).__init__(msg) - - -class VMHost(object): - - def __init__(self, user, address, disk_path): - self.user = user - self.address = address - self.disk_path = disk_path - - @property - def ssh_host(self): - return '{user}@{address}'.format(user=self.user, address=self.address) - - def runcmd(self, *args, **kwargs): - cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) - - -class DeployedSystemInstance(object): - - def __init__(self, deployment, config, host_machine, vm_id, rootfs_path, - ip_addr, hostname): - self.deployment = deployment - self.config = config - self.ip_address = ip_addr - self.host_machine = host_machine - self.vm_id = vm_id - self.rootfs_path = rootfs_path - self.hostname = hostname - - @property - def ssh_host(self): - # TODO: Stop assuming we ssh into test instances as root - return 'root@{host}'.format(host=self.ip_address) - - def runcmd(self, argv, chdir='.', **kwargs): - ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] - cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] - cmd += argv - ssh_cmd.append(' '.join(map(pipes.quote, cmd))) - return cliapp.runcmd(ssh_cmd, **kwargs) - - def _wait_for_dhcp(self, timeout): - '''Block until given hostname resolves successfully. - - Raises TimeoutError if the hostname has not appeared in 'timeout' - seconds. - - ''' - start_time = time.time() - while True: - try: - socket.gethostbyname(self.ip_address) - return - except socket.gaierror: - pass - if time.time() > start_time + timeout: - raise TimeoutError("Host %s did not appear after %i seconds" % - (self.ip_address, timeout)) - time.sleep(0.5) - - def _wait_for_ssh(self, timeout): - """Wait until the deployed VM is responding via SSH""" - start_time = time.time() - while True: - try: - self.runcmd(['true'], stdin=None, stdout=None, stderr=None) - return - except cliapp.AppException: - # TODO: Stop assuming the ssh part of the command is what failed - if time.time() > start_time + timeout: - raise TimeoutError("%s sshd did not start after %i seconds" - % (self.ip_address, timeout)) - time.sleep(0.5) - - def _wait_for_cloud_init(self, timeout): - """Wait until cloud init has resized the disc""" - start_time = time.time() - while True: - try: - out = self.runcmd(['sh', '-c', - 'test -e "$1" && echo exists || echo does not exist', - '-', - '/root/cloud-init-finished']) - except: - import traceback - traceback.print_exc() - raise - if out.strip() == 'exists': - return - if time.time() > start_time + timeout: - raise TimeoutError("Disc size not increased after %i seconds" - % (timeout)) - time.sleep(3) - - def wait_until_online(self, timeout=120): - self._wait_for_dhcp(timeout) - self._wait_for_ssh(timeout) - self._wait_for_cloud_init(timeout) - print "Test system %s ready to run tests." % (self.hostname) - - def delete(self): - # Stop and remove VM - print "Deleting %s test instance" % (self.hostname) - try: - cliapp.runcmd(['nova', 'delete', self.hostname]) - except cliapp.AppException as e: - # TODO: Stop assuming that delete failed because the instance - # wasn't running - print "- Failed" - pass - print "Deleting %s test disc image" % (self.hostname) - try: - cliapp.runcmd(['nova', 'image-delete', self.hostname]) - except cliapp.AppException as e: - # TODO: Stop assuming that image-delete failed because it was - # already removed - print "- Failed" - pass - - -class Deployment(object): - - def __init__(self, cluster_path, name, deployment_config, - host_machine, net_id): - self.cluster_path = cluster_path - self.name = name - self.deployment_config = deployment_config - self.host_machine = host_machine - self.net_id = net_id - - @staticmethod - def _ssh_host_key_exists(hostname): - """Check if an ssh host key exists in known_hosts""" - if not os.path.exists('/root/.ssh/known_hosts'): - return False - with open('/root/.ssh/known_hosts', 'r') as known_hosts: - return any(line.startswith(hostname) for line in known_hosts) - - def _update_known_hosts(self): - if not self._ssh_host_key_exists(self.host_machine.address): - with open('/root/.ssh/known_hosts', 'a') as known_hosts: - cliapp.runcmd(['ssh-keyscan', self.host_machine.address], - stdout=known_hosts) - - @staticmethod - def _generate_sshkey_config(tempdir, config): - manifest = os.path.join(tempdir, 'manifest') - with open(manifest, 'w') as f: - f.write('0040700 0 0 /root/.ssh\n') - f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') - authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') - os.makedirs(os.path.dirname(authkeys)) - with open(authkeys, 'w') as auth_f: - with open('/root/.ssh/id_rsa.pub', 'r') as key_f: - shutil.copyfileobj(key_f, auth_f) - - install_files = shlex.split(config.get('INSTALL_FILES', '')) - install_files.append(manifest) - yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) - - def deploy(self): - self._update_known_hosts() - - hostname = str(uuid.uuid4()) - vm_id = hostname - image_base = self.host_machine.disk_path - rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, - hostname=hostname) - loc = 'http://{ssh_host}:5000/v2.0'.format( - ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) - - options = { - 'type': 'openstack', - 'location': loc, - 'HOSTNAME': hostname, - 'DISK_SIZE': '5G', - 'RAM_SIZE': '2G', - 'VERSION_LABEL': 'release-test', - 'OPENSTACK_USER': os.environ['OS_USERNAME'], - 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'], - 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'], - 'OPENSTACK_IMAGENAME': hostname, - 'CLOUD_INIT': 'yes', - 'KERNEL_ARGS': 'console=tty0 console=ttyS0', - } - - tempdir = tempfile.mkdtemp() - try: - options.update( - self._generate_sshkey_config(tempdir, - self.deployment_config)) - - # Deploy the image to openstack - args = ['morph', 'deploy', self.cluster_path, self.name] - for k, v in options.iteritems(): - args.append('%s.%s=%s' % (self.name, k, v)) - cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) - - config = dict(self.deployment_config) - config.update(options) - - # Boot an instance from the image - args = ['nova', 'boot', - '--flavor', 'm1.medium', - '--image', hostname, - '--user-data', '/usr/lib/mason/os-init-script', - '--nic', "net-id=%s" % (self.net_id), - hostname] - output = cliapp.runcmd(args) - - # Print nova boot output, with adminPass line removed - output_lines = output.split('\n') - for line in output_lines: - if line.find('adminPass') != -1: - password_line = line - output_lines.remove(password_line) - output = '\n'.join(output_lines) - print output - - # Get ip address from nova list - nl = NovaList() - ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) - print "IP address for instance %s: %s" % (hostname, ip_addr) - - return DeployedSystemInstance(self, config, self.host_machine, - vm_id, rootpath, ip_addr, hostname) - finally: - shutil.rmtree(tempdir) - - -class ReleaseApp(cliapp.Application): - - """Cliapp application which handles automatic builds and tests""" - - def add_settings(self): - """Add the command line options needed""" - group_main = 'Program Options' - self.settings.string_list(['deployment-host'], - 'ARCH:HOST:PATH that VMs can be deployed to', - default=None, - group=group_main) - self.settings.string(['trove-host'], - 'Address of Trove for test systems to build from', - default=None, - group=group_main) - self.settings.string(['trove-id'], - 'ID of Trove for test systems to build from', - default=None, - group=group_main) - self.settings.string(['build-ref-prefix'], - 'Prefix of build branches for test systems', - default=None, - group=group_main) - self.settings.string(['net-id'], - 'Openstack network ID', - default=None, - group=group_main) - - @staticmethod - def _run_tests(instance, system_path, system_morph, - (trove_host, trove_id, build_ref_prefix), - morph_helper, systems): - instance.wait_until_online() - - tests = [] - def baserock_build_test(instance): - instance.runcmd(['git', 'config', '--global', 'user.name', - 'Test Instance of %s' % instance.deployment.name]) - instance.runcmd(['git', 'config', '--global', 'user.email', - 'ci-test@%s' % instance.config['HOSTNAME']]) - instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', - '/src/tmp']) - def morph_cmd(*args, **kwargs): - # TODO: decide whether to use cached artifacts or not by - # adding --artifact-cache-server= --cache-server= - argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', - '--tempdir=/src/tmp', '--log-max=100M', - '--trove-host', trove_host, '--trove-id', trove_id, - '--build-ref-prefix', build_ref_prefix] - argv.extend(args) - instance.runcmd(argv, **kwargs) - - repo = morph_helper.sb.root_repository_url - ref = morph_helper.defs_repo.HEAD - sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref) - morph_cmd('init', '/src/ws') - chdir = '/src/ws' - - morph_cmd('checkout', repo, ref, chdir=chdir) - # TODO: Add a morph subcommand that gives the path to the root repository. - repo_path = os.path.relpath( - morph_helper.sb.get_git_directory_name(repo), - morph_helper.sb.root_directory) - chdir = os.path.join(chdir, ref, repo_path) - - instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) - print 'Building test systems for {sys}'.format(sys=system_path) - for to_build_path, to_build_morph in systems.iteritems(): - if to_build_morph['arch'] == system_morph['arch']: - print 'Test building {path}'.format(path=to_build_path) - morph_cmd('build', to_build_path, chdir=chdir, - stdin=None, stdout=None, stderr=None) - print 'Finished Building test systems' - - def python_smoke_test(instance): - instance.runcmd(['python', '-c', 'print "Hello World"']) - - # TODO: Come up with a better way of determining which tests to run - if 'devel' in system_path: - tests.append(baserock_build_test) - else: - tests.append(python_smoke_test) - - for test in tests: - test(instance) - - def deploy_and_test_systems(self, cluster_path, - deployment_hosts, build_test_config, - net_id): - """Run the deployments and tests""" - - version = 'release-test' - - morph_helper = MorphologyHelper() - cluster_morph = morph_helper.load_morphology(cluster_path) - systems = dict(morph_helper.load_cluster_systems(cluster_morph)) - - for system_path, deployment_name, deployment_config in \ - morph_helper.iterate_cluster_deployments(cluster_morph): - - system_morph = systems[system_path] - # We can only test systems in KVM that have a BSP - if not any('bsp' in si['morph'] for si in system_morph['strata']): - continue - - # We can only test systems in KVM that we have a host for - if system_morph['arch'] not in deployment_hosts: - continue - host_machine = deployment_hosts[system_morph['arch']] - deployment = Deployment(cluster_path, deployment_name, - deployment_config, host_machine, - net_id) - - instance = deployment.deploy() - try: - self._run_tests(instance, system_path, system_morph, - build_test_config, morph_helper, systems) - finally: - instance.delete() - - def process_args(self, args): - """Process the command line args and kick off the builds/tests""" - if self.settings['build-ref-prefix'] is None: - self.settings['build-ref-prefix'] = ( - os.path.join(self.settings['trove-id'], 'builds')) - for setting in ('deployment-host', 'trove-host', - 'trove-id', 'build-ref-prefix', 'net-id'): - self.settings.require(setting) - - deployment_hosts = {} - for host_config in self.settings['deployment-host']: - arch, address = host_config.split(':', 1) - user, address = address.split('@', 1) - address, disk_path = address.split(':', 1) - if user == '': - user = 'root' - # TODO: Don't assume root is the user with deploy access - deployment_hosts[arch] = VMHost(user, address, disk_path) - - build_test_config = (self.settings['trove-host'], - self.settings['trove-id'], - self.settings['build-ref-prefix']) - - if len(args) != 1: - raise cliapp.AppException('Usage: release-test CLUSTER') - cluster_path = morphlib.util.sanitise_morphology_path(args[0]) - self.deploy_and_test_systems(cluster_path, deployment_hosts, - build_test_config, - self.settings['net-id']) - - -if __name__ == '__main__': - ReleaseApp().run() diff --git a/scripts/release-upload b/scripts/release-upload deleted file mode 100755 index f8b3337c..00000000 --- a/scripts/release-upload +++ /dev/null @@ -1,473 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''Upload and publish Baserock binaries for a release. - -This utility is used for the Baserock release process. See -http://wiki.baserock.org/guides/release-process/ for details on the -release process. - -This utility uploads two sets of binaries: - -* The build artifacts (built chunks and strata) used to construct the - systems being released. The systems are found in `release.morph` and - the artifacts from the Trove used to prepare the release. They get - uploaded to a public Trove (by default git.baserock.org). If they're - the same Trove, then nothing happens. - -* The released system images (disk images, tar archives, etc) - specified in `release.morph` get uploaded to a download server (by - default download.baserock.org). - -''' - - -import json -import logging -import os -import pwd -import shutil -import sys -import urllib -import urllib2 -import urlparse - -import cliapp -import yaml - -import morphlib - -class ReleaseUploader(cliapp.Application): - - def add_settings(self): - group = 'Release upload settings' - - local_username = self.get_local_username() - - self.settings.string( - ['build-trove-host'], - 'get build artifacts from Trove at ADDRESS', - metavar='ADDRESS', - group=group) - - self.settings.string( - ['public-trove-host'], - 'publish build artifacts on Trove at ADDRESS', - metavar='ADDRESS', - default='git.baserock.org', - group=group) - - self.settings.string( - ['public-trove-username'], - 'log into public trove as USER', - metavar='USER', - default=local_username, - group=group) - - self.settings.string( - ['public-trove-artifact-dir'], - 'put published artifacts into DIR', - metavar='DIR', - default='/home/cache/artifacts', - group=group) - - self.settings.string( - ['release-artifact-dir'], - 'get release artifacts from DIR (all files from there)', - metavar='DIR', - default='.', - group=group) - - self.settings.string( - ['download-server-address'], - 'publish release artifacts on server at ADDRESS', - metavar='ADDRESS', - default='download.baserock.org', - group=group) - - self.settings.string( - ['download-server-username'], - 'log into download server as USER', - metavar='USER', - default=local_username, - group=group) - - self.settings.string( - ['download-server-private-dir'], - 'use DIR as the temporary location for uploaded release ' - 'artifacts', - metavar='DIR', - default='/srv/download.baserock.org/baserock/.publish-temp', - group=group) - - self.settings.string( - ['download-server-public-dir'], - 'put published release artifacts in DIR', - metavar='DIR', - default='/srv/download.baserock.org/baserock', - group=group) - - self.settings.string( - ['local-build-artifacts-dir'], - 'keep build artifacts to be uploaded temporarily in DIR', - metavar='DIR', - default='build-artifacts', - group=group) - - self.settings.string( - ['morph-cmd'], - 'run FILE to invoke morph', - metavar='FILE', - default='morph', - group=group) - - self.settings.string_list( - ['arch'], - 'Upload files from morphologies of ARCH', - metavar='ARCH', - default=[], - group=group) - - self.settings.boolean( - ['upload-build-artifacts'], - 'upload build artifacts?', - default=True) - - self.settings.boolean( - ['upload-release-artifacts'], - 'upload release artifacts (disk images etc)?', - default=True) - - def get_local_username(self): - uid = os.getuid() - return pwd.getpwuid(uid)[0] - - def process_args(self, args): - if len(args) != 1: - raise cliapp.AppException('Usage: release-upload CLUSTER') - cluster_morphology_path = args[0] - self.status(msg='Uploading and publishing Baserock release') - - if self.settings['upload-build-artifacts']: - self.publish_build_artifacts(cluster_morphology_path) - else: - self.status( - msg='Not uploading build artifacts ' - '(upload-build-artifacts set to false') - - if self.settings['upload-release-artifacts']: - self.publish_release_artifacts() - else: - self.status( - msg='Not uploading release artifacts ' - '(upload-release-artifacts set to false') - - def publish_build_artifacts(self, cluster_morphology_path): - publisher = BuildArtifactPublisher(self.settings, self.status) - publisher.publish_build_artifacts(cluster_morphology_path) - self.status(msg='Build artifacts have been published') - - def publish_release_artifacts(self): - publisher = ReleaseArtifactPublisher(self.settings, self.status) - publisher.publish_release_artifacts() - self.status(msg='Release artifacts have been published') - - def status(self, msg, **kwargs): - formatted = msg.format(**kwargs) - logging.info(formatted) - sys.stdout.write(formatted + '\n') - sys.stdout.flush() - - -class BuildArtifactPublisher(object): - - '''Publish build artifacts related to the release.''' - - def __init__(self, settings, status): - self.settings = settings - self.status = status - - def publish_build_artifacts(self, cluster_path): - artifact_basenames = self.list_build_artifacts_for_release(cluster_path) - self.status( - msg='Found {count} build artifact files in release', - count=len(artifact_basenames)) - - to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( - artifact_basenames) - - logging.debug('List of artifacts (basenames) to upload (without already uploaded):') - for i, basename in enumerate(to_be_uploaded): - logging.debug(' {0}: {1}'.format(i, basename)) - logging.debug('End of artifact list (to_be_uploaded)') - - self.status( - msg='Need to fetch locally, then upload {count} build artifacts', - count=len(to_be_uploaded)) - - self.upload_build_artifacts_to_public_trove(to_be_uploaded) - - def list_build_artifacts_for_release(self, cluster_morphology_path): - self.status(msg='Find build artifacts included in release') - - # FIXME: These are hardcoded for simplicity. They would be - # possible to deduce automatically from the workspace, but - # that can happen later. - repo = 'file://%s' % os.path.abspath('.') - ref = 'HEAD' - - argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', - '--repo', repo, '--ref', ref] - argv += self.find_system_morphologies(cluster_morphology_path) - output = cliapp.runcmd(argv) - basenames = output.splitlines() - logging.debug('List of build artifacts in release:') - for basename in basenames: - logging.debug(' {0}'.format(basename)) - logging.debug('End of list of build artifacts in release') - - return basenames - - def find_system_morphologies(self, cluster_morphology_path): - cluster = self.load_cluster_morphology(cluster_morphology_path) - system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) - if self.settings['arch']: - system_dicts = self.choose_systems_for_wanted_architectures( - system_dicts, self.settings['arch']) - return [sd['morph'] for sd in system_dicts] - - def load_cluster_morphology(self, pathname): - with open(pathname) as f: - return yaml.load(f) - - def find_systems_in_parsed_cluster_morphology(self, cluster): - return cluster['systems'] - - def choose_systems_for_wanted_architectures(self, system_dicts, archs): - return [ - sd - for sd in system_dicts - if self.system_is_for_wanted_arch(sd, archs)] - - def system_is_for_wanted_arch(self, system_dict, archs): - morph = self.load_system_morphology(system_dict) - return morph['arch'] in archs - - def load_system_morphology(self, system_dict): - pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) - return self.load_morphology_from_named_file(pathname) - - def load_morphology_from_named_file(self, pathname): - finder = self.get_morphology_finder_for_root_repository() - morphology_text = finder.read_file(pathname) - loader = morphlib.morphloader.MorphologyLoader() - return loader.load_from_string(morphology_text) - - def get_morphology_finder_for_root_repository(self): - definitions_repo = morphlib.definitions_repo.open( - '.', search_for_root=True) - return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) - - def filter_away_build_artifacts_on_public_trove(self, basenames): - result = [] - logging.debug('Filtering away already existing artifacts:') - for basename, exists in self.query_public_trove_for_artifacts(basenames): - logging.debug(' {0}: {1}'.format(basename, exists)) - if not exists: - result.append(basename) - logging.debug('End of filtering away') - return result - - def query_public_trove_for_artifacts(self, basenames): - host = self.settings['public-trove-host'] - - # FIXME: This could use - # contextlib.closing(urllib2.urlopen(url, data=data) instead - # of explicit closing. - url = 'http://{host}:8080/1.0/artifacts'.format(host=host) - data = json.dumps(basenames) - f = urllib2.urlopen(url, data=data) - obj = json.load(f) - return obj.items() - - def upload_build_artifacts_to_public_trove(self, basenames): - self.download_artifacts_locally(basenames) - self.upload_artifacts_to_public_trove(basenames) - - def download_artifacts_locally(self, basenames): - dirname = self.settings['local-build-artifacts-dir'] - self.create_directory_if_missing(dirname) - for i, basename in enumerate(basenames): - url = self.construct_artifact_url(basename) - pathname = os.path.join(dirname, basename) - if not os.path.exists(pathname): - self.status( - msg='Downloading {i}/{total} {basename}', - basename=repr(basename), i=i, total=len(basenames)) - self.download_from_url(url, dirname, pathname) - - def create_directory_if_missing(self, dirname): - if not os.path.exists(dirname): - os.makedirs(dirname) - - def construct_artifact_url(self, basename): - scheme = 'http' - netloc = '{host}:8080'.format(host=self.settings['build-trove-host']) - path = '/1.0/artifacts' - query = 'filename={0}'.format(urllib.quote_plus(basename)) - fragment = '' - components = (scheme, netloc, path, query, fragment) - return urlparse.urlunsplit(components) - - def download_from_url(self, url, dirname, pathname): - logging.info( - 'Downloading {url} to {pathname}'.format( - url=url, pathname=pathname)) - with open(pathname, 'wb') as output: - try: - incoming = urllib2.urlopen(url) - shutil.copyfileobj(incoming, output) - incoming.close() - except urllib2.HTTPError as e: - if pathname.endswith('.meta'): - return - self.status( - msg="ERROR: Can't download {url}: {explanation}", - url=url, - explanation=str(e)) - os.remove(pathname) - raise - - def upload_artifacts_to_public_trove(self, basenames): - self.status( - msg='Upload build artifacts to {trove}', - trove=self.settings['public-trove-host']) - rsync_files_to_server( - self.settings['local-build-artifacts-dir'], - basenames, - self.settings['public-trove-username'], - self.settings['public-trove-host'], - self.settings['public-trove-artifact-dir']) - set_permissions_on_server( - self.settings['public-trove-username'], - self.settings['public-trove-host'], - self.settings['public-trove-artifact-dir'], - basenames) - -class ReleaseArtifactPublisher(object): - - '''Publish release artifacts for a release.''' - - def __init__(self, settings, status): - self.settings = settings - self.status = status - - def publish_release_artifacts(self): - files = self.list_release_artifacts() - if files: - self.upload_release_artifacts_to_private_dir(files) - self.move_release_artifacts_to_public_dir(files) - self.create_symlinks_to_new_release_artifacts(files) - - def list_release_artifacts(self): - self.status(msg='Find release artifacts to publish') - return os.listdir(self.settings['release-artifact-dir']) - - def upload_release_artifacts_to_private_dir(self, files): - self.status(msg='Upload release artifacts to private directory') - path = self.settings['download-server-private-dir'] - self.create_directory_on_download_server(path) - self.rsync_files_to_download_server(files, path) - - def create_directory_on_download_server(self, path): - user = self.settings['download-server-username'] - host = self.settings['download-server-address'] - self.status(msg='Create {host}:{path}', host=host, path=path) - target = '{user}@{host}'.format(user=user, host=host) - cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) - - def rsync_files_to_download_server(self, files, path): - self.status(msg='Upload release artifacts to download server') - rsync_files_to_server( - self.settings['release-artifact-dir'], - files, - self.settings['download-server-username'], - self.settings['download-server-address'], - path) - set_permissions_on_server( - self.settings['download-server-username'], - self.settings['download-server-address'], - path, - files) - - def move_release_artifacts_to_public_dir(self, files): - self.status(msg='Move release artifacts to public directory') - private_dir = self.settings['download-server-private-dir'] - public_dir = self.settings['download-server-public-dir'] - self.create_directory_on_download_server(public_dir) - - # Move just the contents of the private dir, not the dir - # itself (-mindepth). Avoid overwriting existing files (mv - # -n). - argv = ['find', private_dir, '-mindepth', '1', - '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] - - target = '{user}@{host}'.format( - user=self.settings['download-server-username'], - host=self.settings['download-server-address']) - cliapp.ssh_runcmd(target, argv) - - def create_symlinks_to_new_release_artifacts(self, files): - self.status(msg='FIXME: Create symlinks to new releas artifacts') - - -def rsync_files_to_server( - source_dir, source_filenames, user, host, target_dir): - - if not source_filenames: - return - - argv = [ - 'rsync', - '-a', - '--progress', - '--partial', - '--human-readable', - '--sparse', - '--protect-args', - '-0', - '--files-from=-', - source_dir, - '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), - ] - - files_list = '\0'.join(filename for filename in source_filenames) - cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) - - -def set_permissions_on_server(user, host, target_dir, filenames): - # If we have no files, we can't form a valid command to run on the server - if not filenames: - return - target = '{user}@{host}'.format(user=user, host=host) - argv = ['xargs', '-0', 'chmod', '0644'] - files_list = ''.join( - '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) - cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) - - -ReleaseUploader(description=__doc__).run() diff --git a/scripts/release-upload.test.conf b/scripts/release-upload.test.conf deleted file mode 100644 index 13227983..00000000 --- a/scripts/release-upload.test.conf +++ /dev/null @@ -1,10 +0,0 @@ -[config] -download-server-address = localhost -download-server-private-dir = /tmp/private -download-server-public-dir = /tmp/public -build-trove-host = ct-mcr-1.ducie.codethink.co.uk -public-trove-host = localhost -public-trove-username = root -public-trove-artifact-dir = /tmp/artifacts -release-artifact-dir = t.release-files -morph-cmd = /home/root/git-morph diff --git a/scripts/scriptslib.py b/scripts/scriptslib.py deleted file mode 100644 index 53c6ca8e..00000000 --- a/scripts/scriptslib.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# Small library of useful things for the scripts that live here. - -import yaml -import subprocess -import os -import sys - -aliases = { - 'baserock:': 'git://%(trove)s/baserock/', - 'freedesktop:': 'git://anongit.freedesktop.org/', - 'github:': 'git://github.com/', - 'gnome:': 'git://git.gnome.org/', - 'upstream:': 'git://%(trove)s/delta/' -} - -def parse_repo_alias(repo, trove_host='git.baserock.org'): - global aliases - remote = repo[:repo.find(':') + 1] - aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()} - try: - return repo.replace(remote, aliases[remote]) - except KeyError as e: - raise Exception("Unknown repo-alias \"%s\"" % repo) - -def definitions_root(): - return subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"]).strip() - -def load_yaml_file(yaml_file): - with open(yaml_file, 'r') as f: - return yaml.safe_load(f) - - -class BaserockMeta(object): - '''An object representing Baserock metadata contained in a Baserock - system image, for available metadata formats''' - - def __init__(self): - self.metas = {} - - def get_each(self): - '''Yield an iterable for the whole list of metas''' - for key in self.metas: - yield self.metas[key] - - def get_name(self, name): - '''Yield an iterable of metadata matched by name, e.g. `bash`''' - for key in self.metas: - if self.metas[key]['source-name'] == name: - yield self.metas[key] - - def import_meta(self, meta_text): - importers = (self.import_meta_ybd, - self.import_meta_morph) - - for i in importers: - try: - i(meta_text) - return - except (KeyError, Exception) as err: - pass - - # Shouldn't get here - sys.stderr.write('Metadata format not recognised.\n' - 'Error:\n') - raise err - - def import_meta_morph(self, meta_text): - self._add_meta(yaml.load(meta_text)) - - def import_meta_ybd(self, meta_text): - source = yaml.load(meta_text) - - null = '0' * 32 - - if 'configuration' in source: - # This is the deployment metadata, ignore - return - elif 'repo' not in source: - kind = 'stratum' - contents = 'components' - source['repo'] = 'upstream:definitions' - source['ref'] = null # No ref info - else: - kind = 'chunk' - contents = 'components' - - repo = parse_repo_alias(source['repo']) - source_name = '-'.join( - source['products'][0]['artifact'].split('-')[:-1]) - - # Needed until YBD provides cache-key in metadata - if not 'cache-key' in source: - source['cache-key'] = null - - for product in source['products']: - - self._add_meta({ - 'kind': kind, - 'source-name': source_name, - 'artifact-name': product['artifact'], - 'contents': product[contents], - 'repo': repo, - 'repo-alias': source['repo'], - 'sha1': source['ref'], - 'original_ref': source['ref'], - 'cache-key': source['cache-key'] - }) - - def _add_meta(self, meta_dict): - '''Validate and add a meta''' - - ignore = ('configuration', - 'system-artifact-name') - - for i in ignore: - if i in meta_dict: - return - - required_fields = ('repo', 'sha1', 'contents') - for f in required_fields: - if not f in meta_dict: - raise Exception('Metadata format not recognised, no ' - 'value for \'%s\'. Data: \'%s\''% (f, str(meta_dict))) - - self.metas[meta_dict['artifact-name']] = meta_dict - - -def meta_load_from_dir(meta_dir_path): - '''Read Baserock metadata from a directory''' - - files = [f for f in os.listdir(meta_dir_path) - if os.path.isfile(os.path.join(meta_dir_path, f))] - - meta = BaserockMeta() - for f in files: - if f.endswith('.meta'): - meta.import_meta( - open(os.path.join(meta_dir_path, f), 'r').read()) - - return meta diff --git a/scripts/yaml-jsonschema b/scripts/yaml-jsonschema deleted file mode 100755 index 64f52a79..00000000 --- a/scripts/yaml-jsonschema +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''A tool to validate YAML files against the JSON-Schema schemas. - -This wraps Python `jsonschema` module so that YAML schemas can be understood -and YAML data can be validated. - -Usage: yaml-jsonschema SCHEMA INPUT1 [INPUT2, ...] - -''' - - -import jsonschema -import yaml - -import sys - - -schema_file = sys.argv[1] -input_files = sys.argv[2:] - - -with open(schema_file) as f: - schema = yaml.load(f) - - -for input_file in input_files: - with open(input_file) as f: - data = yaml.load(f) - - try: - jsonschema.validate(data, schema) - print("%s: valid" % input_file) - except jsonschema.ValidationError as e: - # Print 'e' instead of 'e.message' for more information! - print("%s: %s" % (input_file, e.message)) -- cgit v1.2.1