From 19f6fc68d4c44869e266f97c8667d13971f1d5d9 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 19 Feb 2020 09:55:34 -0800 Subject: Relocate Shippable tools. (#67556) * Move Shippable tools to hacking directory. These limits the `test/utils/shippable/` directory to scripts required for CI. * Fix `test/utils/shippable/` file classification. * Update package-data sanity test. --- hacking/shippable/download.py | 347 ++++++++++++++++++++++++++++++++++++++++++ hacking/shippable/run.py | 150 ++++++++++++++++++ 2 files changed, 497 insertions(+) create mode 100755 hacking/shippable/download.py create mode 100755 hacking/shippable/run.py (limited to 'hacking') diff --git a/hacking/shippable/download.py b/hacking/shippable/download.py new file mode 100755 index 0000000000..5449ee7e6f --- /dev/null +++ b/hacking/shippable/download.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK + +# (c) 2016 Red Hat, Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +"""CLI tool for downloading results from Shippable CI runs.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# noinspection PyCompatibility +import argparse +import json +import os +import re +import sys + +import requests + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def main(): + """Main program body.""" + args = parse_args() + download_run(args) + + +def parse_args(): + """Parse and return args.""" + api_key = get_api_key() + + parser = argparse.ArgumentParser(description='Download results from a Shippable run.') + + parser.add_argument('run_id', + metavar='RUN', + help='shippable run id, run url or run name formatted as: account/project/run_number') + + parser.add_argument('-v', '--verbose', + dest='verbose', + action='store_true', + help='show what is being downloaded') + + parser.add_argument('-t', '--test', + dest='test', + action='store_true', + help='show what would be downloaded without downloading') + + parser.add_argument('--key', + dest='api_key', + default=api_key, + required=api_key is None, + help='api key for accessing Shippable') + + parser.add_argument('--console-logs', + action='store_true', + help='download console logs') + + parser.add_argument('--test-results', + action='store_true', + help='download test results') + + parser.add_argument('--coverage-results', + action='store_true', + help='download code coverage results') + + parser.add_argument('--job-metadata', + action='store_true', + help='download job metadata') + + parser.add_argument('--run-metadata', + action='store_true', + help='download run metadata') + + parser.add_argument('--all', + action='store_true', + help='download everything') + + parser.add_argument('--job-number', + metavar='N', + action='append', + type=int, + help='limit downloads to the given job number') + + if argcomplete: + argcomplete.autocomplete(parser) + + args = parser.parse_args() + + old_runs_prefix = 'https://app.shippable.com/runs/' + + if args.run_id.startswith(old_runs_prefix): + args.run_id = args.run_id[len(old_runs_prefix):] + + if args.all: + args.console_logs = True + args.test_results = True + args.coverage_results = True + args.job_metadata = True + args.run_metadata = True + + selections = ( + args.console_logs, + args.test_results, + args.coverage_results, + args.job_metadata, + args.run_metadata, + ) + + if not any(selections): + parser.error('At least one download option is required.') + + return args + + +def download_run(args): + """Download a Shippable run.""" + headers = dict( + Authorization='apiToken %s' % args.api_key, + ) + + match = re.search( + r'^https://app.shippable.com/github/(?P[^/]+)/(?P[^/]+)/runs/(?P[0-9]+)(?:/summary|(/(?P[0-9]+)))?$', + args.run_id) + + if not match: + match = re.search(r'^(?P[^/]+)/(?P[^/]+)/(?P[0-9]+)$', args.run_id) + + if match: + account = match.group('account') + project = match.group('project') + run_number = int(match.group('run_number')) + job_number = int(match.group('job_number')) if match.group('job_number') else None + + if job_number: + if args.job_number: + sys.exit('ERROR: job number found in url and specified with --job-number') + + args.job_number = [job_number] + + url = 'https://api.shippable.com/projects' + response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + project_id = response.json()[0]['id'] + + url = 'https://api.shippable.com/runs?projectIds=%s&runNumbers=%s' % (project_id, run_number) + + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + run = [run for run in response.json() if run['runNumber'] == run_number][0] + + args.run_id = run['id'] + elif re.search('^[a-f0-9]+$', args.run_id): + url = 'https://api.shippable.com/runs/%s' % args.run_id + + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + run = response.json() + + account = run['subscriptionOrgName'] + project = run['projectName'] + run_number = run['runNumber'] + else: + sys.exit('ERROR: invalid run: %s' % args.run_id) + + output_dir = '%s/%s/%s' % (account, project, run_number) + + response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + jobs = sorted(response.json(), key=lambda job: int(job['jobNumber'])) + + if not args.test: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + if args.run_metadata: + path = os.path.join(output_dir, 'run.json') + contents = json.dumps(run, sort_keys=True, indent=4) + + if args.verbose or args.test: + print(path) + + if not args.test: + with open(path, 'w') as metadata_fd: + metadata_fd.write(contents) + + download_jobs(args, jobs, headers, output_dir) + + +def download_jobs(args, jobs, headers, output_dir): + """Download Shippable jobs.""" + for j in jobs: + job_id = j['id'] + job_number = j['jobNumber'] + + if args.job_number and job_number not in args.job_number: + continue + + if args.job_metadata: + path = os.path.join(output_dir, '%s/job.json' % job_number) + contents = json.dumps(j, sort_keys=True, indent=4).encode('utf-8') + + if args.verbose or args.test: + print(path) + + if not args.test: + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + with open(path, 'wb') as metadata_fd: + metadata_fd.write(contents) + + if args.console_logs: + path = os.path.join(output_dir, '%s/console.log' % job_number) + url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id + download(args, headers, path, url, is_json=False) + + if args.test_results: + path = os.path.join(output_dir, '%s/test.json' % job_number) + url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id + download(args, headers, path, url) + extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number)) + + if args.coverage_results: + path = os.path.join(output_dir, '%s/coverage.json' % job_number) + url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id + download(args, headers, path, url) + extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number)) + + +def extract_contents(args, path, output_dir): + """ + :type args: any + :type path: str + :type output_dir: str + """ + if not args.test: + if not os.path.exists(path): + return + + with open(path, 'r') as json_fd: + items = json.load(json_fd) + + for item in items: + contents = item['contents'].encode('utf-8') + path = output_dir + '/' + re.sub('^/*', '', item['path']) + + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + if args.verbose: + print(path) + + if path.endswith('.json'): + contents = json.dumps(json.loads(contents), sort_keys=True, indent=4).encode('utf-8') + + if not os.path.exists(path): + with open(path, 'wb') as output_fd: + output_fd.write(contents) + + +def download(args, headers, path, url, is_json=True): + """ + :type args: any + :type headers: dict[str, str] + :type path: str + :type url: str + :type is_json: bool + """ + if args.verbose or args.test: + print(path) + + if os.path.exists(path): + return + + if not args.test: + response = requests.get(url, headers=headers) + + if response.status_code != 200: + path += '.error' + + if is_json: + content = json.dumps(response.json(), sort_keys=True, indent=4).encode(response.encoding) + else: + content = response.content + + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + with open(path, 'wb') as content_fd: + content_fd.write(content) + + +def get_api_key(): + """ + rtype: str + """ + key = os.environ.get('SHIPPABLE_KEY', None) + + if key: + return key + + path = os.path.join(os.environ['HOME'], '.shippable.key') + + try: + with open(path, 'r') as key_fd: + return key_fd.read().strip() + except IOError: + return None + + +if __name__ == '__main__': + main() diff --git a/hacking/shippable/run.py b/hacking/shippable/run.py new file mode 100755 index 0000000000..310a7f53f0 --- /dev/null +++ b/hacking/shippable/run.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK + +# (c) 2016 Red Hat, Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +"""CLI tool for starting new Shippable CI runs.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# noinspection PyCompatibility +import argparse +import json +import os + +import requests + +try: + import argcomplete +except ImportError: + argcomplete = None + + +def main(): + """Main program body.""" + args = parse_args() + start_run(args) + + +def parse_args(): + """Parse and return args.""" + api_key = get_api_key() + + parser = argparse.ArgumentParser(description='Start a new Shippable run.') + + parser.add_argument('project', + metavar='account/project', + help='Shippable account/project') + + target = parser.add_mutually_exclusive_group() + + target.add_argument('--branch', + help='branch name') + + target.add_argument('--run', + metavar='ID', + help='Shippable run ID') + + parser.add_argument('--key', + metavar='KEY', + default=api_key, + required=not api_key, + help='Shippable API key') + + parser.add_argument('--env', + nargs=2, + metavar=('KEY', 'VALUE'), + action='append', + help='environment variable to pass') + + if argcomplete: + argcomplete.autocomplete(parser) + + args = parser.parse_args() + + return args + + +def start_run(args): + """Start a new Shippable run.""" + headers = dict( + Authorization='apiToken %s' % args.key, + ) + + # get project ID + + data = dict( + projectFullNames=args.project, + ) + + url = 'https://api.shippable.com/projects' + response = requests.get(url, data, headers=headers) + + if response.status_code != 200: + raise Exception(response.content) + + result = response.json() + + if len(result) != 1: + raise Exception( + 'Received %d items instead of 1 looking for %s in:\n%s' % ( + len(result), + args.project, + json.dumps(result, indent=4, sort_keys=True))) + + project_id = response.json()[0]['id'] + + # new build + + data = dict( + globalEnv=dict((kp[0], kp[1]) for kp in args.env or []) + ) + + if args.branch: + data['branchName'] = args.branch + elif args.run: + data['runId'] = args.run + + url = 'https://api.shippable.com/projects/%s/newBuild' % project_id + response = requests.post(url, json=data, headers=headers) + + if response.status_code != 200: + raise Exception("HTTP %s: %s\n%s" % (response.status_code, response.reason, response.content)) + + print(json.dumps(response.json(), indent=4, sort_keys=True)) + + +def get_api_key(): + """ + rtype: str + """ + key = os.environ.get('SHIPPABLE_KEY', None) + + if key: + return key + + path = os.path.join(os.environ['HOME'], '.shippable.key') + + try: + with open(path, 'r') as key_fd: + return key_fd.read().strip() + except IOError: + return None + + +if __name__ == '__main__': + main() -- cgit v1.2.1