Relocate Shippable tools. (#67556)

* Move Shippable tools to hacking directory.

These limits the `test/utils/shippable/` directory to scripts required for CI.

* Fix `test/utils/shippable/` file classification.

* Update package-data sanity test.

2 files changed, 497 insertions, 0 deletions

diff --git a/hacking/shippable/download.py b/hacking/shippable/download.py
new file mode 100755
index 0000000000..5449ee7e6f
--- /dev/null
+++ b/hacking/shippable/download.py
@@ -0,0 +1,347 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+
+# (c) 2016 Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+"""CLI tool for downloading results from Shippable CI runs."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+# noinspection PyCompatibility
+import argparse
+import json
+import os
+import re
+import sys
+
+import requests
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+def main():
+    """Main program body."""
+    args = parse_args()
+    download_run(args)
+
+
+def parse_args():
+    """Parse and return args."""
+    api_key = get_api_key()
+
+    parser = argparse.ArgumentParser(description='Download results from a Shippable run.')
+
+    parser.add_argument('run_id',
+                        metavar='RUN',
+                        help='shippable run id, run url or run name formatted as: account/project/run_number')
+
+    parser.add_argument('-v', '--verbose',
+                        dest='verbose',
+                        action='store_true',
+                        help='show what is being downloaded')
+
+    parser.add_argument('-t', '--test',
+                        dest='test',
+                        action='store_true',
+                        help='show what would be downloaded without downloading')
+
+    parser.add_argument('--key',
+                        dest='api_key',
+                        default=api_key,
+                        required=api_key is None,
+                        help='api key for accessing Shippable')
+
+    parser.add_argument('--console-logs',
+                        action='store_true',
+                        help='download console logs')
+
+    parser.add_argument('--test-results',
+                        action='store_true',
+                        help='download test results')
+
+    parser.add_argument('--coverage-results',
+                        action='store_true',
+                        help='download code coverage results')
+
+    parser.add_argument('--job-metadata',
+                        action='store_true',
+                        help='download job metadata')
+
+    parser.add_argument('--run-metadata',
+                        action='store_true',
+                        help='download run metadata')
+
+    parser.add_argument('--all',
+                        action='store_true',
+                        help='download everything')
+
+    parser.add_argument('--job-number',
+                        metavar='N',
+                        action='append',
+                        type=int,
+                        help='limit downloads to the given job number')
+
+    if argcomplete:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+
+    old_runs_prefix = 'https://app.shippable.com/runs/'
+
+    if args.run_id.startswith(old_runs_prefix):
+        args.run_id = args.run_id[len(old_runs_prefix):]
+
+    if args.all:
+        args.console_logs = True
+        args.test_results = True
+        args.coverage_results = True
+        args.job_metadata = True
+        args.run_metadata = True
+
+    selections = (
+        args.console_logs,
+        args.test_results,
+        args.coverage_results,
+        args.job_metadata,
+        args.run_metadata,
+    )
+
+    if not any(selections):
+        parser.error('At least one download option is required.')
+
+    return args
+
+
+def download_run(args):
+    """Download a Shippable run."""
+    headers = dict(
+        Authorization='apiToken %s' % args.api_key,
+    )
+
+    match = re.search(
+        r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)(?:/summary|(/(?P<job_number>[0-9]+)))?$',
+        args.run_id)
+
+    if not match:
+        match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id)
+
+    if match:
+        account = match.group('account')
+        project = match.group('project')
+        run_number = int(match.group('run_number'))
+        job_number = int(match.group('job_number')) if match.group('job_number') else None
+
+        if job_number:
+            if args.job_number:
+                sys.exit('ERROR: job number found in url and specified with --job-number')
+
+            args.job_number = [job_number]
+
+        url = 'https://api.shippable.com/projects'
+        response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(response.content)
+
+        project_id = response.json()[0]['id']
+
+        url = 'https://api.shippable.com/runs?projectIds=%s&runNumbers=%s' % (project_id, run_number)
+
+        response = requests.get(url, headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(response.content)
+
+        run = [run for run in response.json() if run['runNumber'] == run_number][0]
+
+        args.run_id = run['id']
+    elif re.search('^[a-f0-9]+$', args.run_id):
+        url = 'https://api.shippable.com/runs/%s' % args.run_id
+
+        response = requests.get(url, headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(response.content)
+
+        run = response.json()
+
+        account = run['subscriptionOrgName']
+        project = run['projectName']
+        run_number = run['runNumber']
+    else:
+        sys.exit('ERROR: invalid run: %s' % args.run_id)
+
+    output_dir = '%s/%s/%s' % (account, project, run_number)
+
+    response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception(response.content)
+
+    jobs = sorted(response.json(), key=lambda job: int(job['jobNumber']))
+
+    if not args.test:
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+    if args.run_metadata:
+        path = os.path.join(output_dir, 'run.json')
+        contents = json.dumps(run, sort_keys=True, indent=4)
+
+        if args.verbose or args.test:
+            print(path)
+
+        if not args.test:
+            with open(path, 'w') as metadata_fd:
+                metadata_fd.write(contents)
+
+    download_jobs(args, jobs, headers, output_dir)
+
+
+def download_jobs(args, jobs, headers, output_dir):
+    """Download Shippable jobs."""
+    for j in jobs:
+        job_id = j['id']
+        job_number = j['jobNumber']
+
+        if args.job_number and job_number not in args.job_number:
+            continue
+
+        if args.job_metadata:
+            path = os.path.join(output_dir, '%s/job.json' % job_number)
+            contents = json.dumps(j, sort_keys=True, indent=4).encode('utf-8')
+
+            if args.verbose or args.test:
+                print(path)
+
+            if not args.test:
+                directory = os.path.dirname(path)
+
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                with open(path, 'wb') as metadata_fd:
+                    metadata_fd.write(contents)
+
+        if args.console_logs:
+            path = os.path.join(output_dir, '%s/console.log' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
+            download(args, headers, path, url, is_json=False)
+
+        if args.test_results:
+            path = os.path.join(output_dir, '%s/test.json' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
+            download(args, headers, path, url)
+            extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number))
+
+        if args.coverage_results:
+            path = os.path.join(output_dir, '%s/coverage.json' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
+            download(args, headers, path, url)
+            extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number))
+
+
+def extract_contents(args, path, output_dir):
+    """
+    :type args: any
+    :type path: str
+    :type output_dir: str
+    """
+    if not args.test:
+        if not os.path.exists(path):
+            return
+
+        with open(path, 'r') as json_fd:
+            items = json.load(json_fd)
+
+            for item in items:
+                contents = item['contents'].encode('utf-8')
+                path = output_dir + '/' + re.sub('^/*', '', item['path'])
+
+                directory = os.path.dirname(path)
+
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                if args.verbose:
+                    print(path)
+
+                if path.endswith('.json'):
+                    contents = json.dumps(json.loads(contents), sort_keys=True, indent=4).encode('utf-8')
+
+                if not os.path.exists(path):
+                    with open(path, 'wb') as output_fd:
+                        output_fd.write(contents)
+
+
+def download(args, headers, path, url, is_json=True):
+    """
+    :type args: any
+    :type headers: dict[str, str]
+    :type path: str
+    :type url: str
+    :type is_json: bool
+    """
+    if args.verbose or args.test:
+        print(path)
+
+    if os.path.exists(path):
+        return
+
+    if not args.test:
+        response = requests.get(url, headers=headers)
+
+        if response.status_code != 200:
+            path += '.error'
+
+        if is_json:
+            content = json.dumps(response.json(), sort_keys=True, indent=4).encode(response.encoding)
+        else:
+            content = response.content
+
+        directory = os.path.dirname(path)
+
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        with open(path, 'wb') as content_fd:
+            content_fd.write(content)
+
+
+def get_api_key():
+    """
+    rtype: str
+    """
+    key = os.environ.get('SHIPPABLE_KEY', None)
+
+    if key:
+        return key
+
+    path = os.path.join(os.environ['HOME'], '.shippable.key')
+
+    try:
+        with open(path, 'r') as key_fd:
+            return key_fd.read().strip()
+    except IOError:
+        return None
+
+
+if __name__ == '__main__':
+    main()
diff --git a/hacking/shippable/run.py b/hacking/shippable/run.py
new file mode 100755
index 0000000000..310a7f53f0
--- /dev/null
+++ b/hacking/shippable/run.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+
+# (c) 2016 Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+"""CLI tool for starting new Shippable CI runs."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+# noinspection PyCompatibility
+import argparse
+import json
+import os
+
+import requests
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+def main():
+    """Main program body."""
+    args = parse_args()
+    start_run(args)
+
+
+def parse_args():
+    """Parse and return args."""
+    api_key = get_api_key()
+
+    parser = argparse.ArgumentParser(description='Start a new Shippable run.')
+
+    parser.add_argument('project',
+                        metavar='account/project',
+                        help='Shippable account/project')
+
+    target = parser.add_mutually_exclusive_group()
+
+    target.add_argument('--branch',
+                        help='branch name')
+
+    target.add_argument('--run',
+                        metavar='ID',
+                        help='Shippable run ID')
+
+    parser.add_argument('--key',
+                        metavar='KEY',
+                        default=api_key,
+                        required=not api_key,
+                        help='Shippable API key')
+
+    parser.add_argument('--env',
+                        nargs=2,
+                        metavar=('KEY', 'VALUE'),
+                        action='append',
+                        help='environment variable to pass')
+
+    if argcomplete:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+
+    return args
+
+
+def start_run(args):
+    """Start a new Shippable run."""
+    headers = dict(
+        Authorization='apiToken %s' % args.key,
+    )
+
+    # get project ID
+
+    data = dict(
+        projectFullNames=args.project,
+    )
+
+    url = 'https://api.shippable.com/projects'
+    response = requests.get(url, data, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception(response.content)
+
+    result = response.json()
+
+    if len(result) != 1:
+        raise Exception(
+            'Received %d items instead of 1 looking for %s in:\n%s' % (
+                len(result),
+                args.project,
+                json.dumps(result, indent=4, sort_keys=True)))
+
+    project_id = response.json()[0]['id']
+
+    # new build
+
+    data = dict(
+        globalEnv=dict((kp[0], kp[1]) for kp in args.env or [])
+    )
+
+    if args.branch:
+        data['branchName'] = args.branch
+    elif args.run:
+        data['runId'] = args.run
+
+    url = 'https://api.shippable.com/projects/%s/newBuild' % project_id
+    response = requests.post(url, json=data, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception("HTTP %s: %s\n%s" % (response.status_code, response.reason, response.content))
+
+    print(json.dumps(response.json(), indent=4, sort_keys=True))
+
+
+def get_api_key():
+    """
+    rtype: str
+    """
+    key = os.environ.get('SHIPPABLE_KEY', None)
+
+    if key:
+        return key
+
+    path = os.path.join(os.environ['HOME'], '.shippable.key')
+
+    try:
+        with open(path, 'r') as key_fd:
+            return key_fd.read().strip()
+    except IOError:
+        return None
+
+
+if __name__ == '__main__':
+    main()