diff options
Diffstat (limited to 'tools/read-dependencies')
| -rwxr-xr-x | tools/read-dependencies | 249 |
1 files changed, 152 insertions, 97 deletions
diff --git a/tools/read-dependencies b/tools/read-dependencies index efa5879c..d6a23c32 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -9,8 +9,9 @@ try: from argparse import ArgumentParser except ImportError: raise RuntimeError( - 'Could not import argparse. Please install python3-argparse ' - 'package to continue') + "Could not import argparse. Please install python3-argparse " + "package to continue" + ) import json import os @@ -18,23 +19,30 @@ import re import subprocess import sys -DEFAULT_REQUIREMENTS = 'requirements.txt' +DEFAULT_REQUIREMENTS = "requirements.txt" # Map the appropriate package dir needed for each distro choice DISTRO_PKG_TYPE_MAP = { - 'centos': 'redhat', - 'eurolinux': 'redhat', - 'miraclelinux': 'redhat', - 'rocky': 'redhat', - 'redhat': 'redhat', - 'debian': 'debian', - 'ubuntu': 'debian', - 'opensuse': 'suse', - 'suse': 'suse' + "centos": "redhat", + "eurolinux": "redhat", + "miraclelinux": "redhat", + "rocky": "redhat", + "redhat": "redhat", + "debian": "debian", + "ubuntu": "debian", + "opensuse": "suse", + "opensuse-leap": "suse", + "opensuse-microos": "suse", + "opensuse-tumbleweed": "suse", + "sle_hpc": "suse", + "sle-micro": "suse", + "sles": "suse", + "suse": "suse", } MAYBE_RELIABLE_YUM_INSTALL = [ - 'sh', '-c', + "sh", + "-c", """ error() { echo "$@" 1>&2; } configure_repos_for_proxy_use() { @@ -61,42 +69,37 @@ MAYBE_RELIABLE_YUM_INSTALL = [ yum install --cacheonly --assumeyes "$@" configure_repos_for_proxy_use """, - 'reliable-yum-install'] + "reliable-yum-install", +] ZYPPER_INSTALL = [ - 'zypper', '--non-interactive', '--gpg-auto-import-keys', 'install', - '--auto-agree-with-licenses'] - -DRY_DISTRO_INSTALL_PKG_CMD = { - 'rocky': ['yum', 'install', '--assumeyes'], - 'centos': ['yum', 'install', '--assumeyes'], - 'eurolinux': ['yum', 'install', '--assumeyes'], - 'miraclelinux': ['yum', 'install', '--assumeyes'], - 'redhat': ['yum', 'install', '--assumeyes'], + "zypper", + "--non-interactive", + "--gpg-auto-import-keys", + "install", + "--auto-agree-with-licenses", +] + +DRYRUN_DISTRO_INSTALL_PKG_CMD = { + "redhat": ["yum", "install", "--assumeyes"], } DISTRO_INSTALL_PKG_CMD = { - 'rocky': MAYBE_RELIABLE_YUM_INSTALL, - 'eurolinux': MAYBE_RELIABLE_YUM_INSTALL, - 'miraclelinux': MAYBE_RELIABLE_YUM_INSTALL, - 'centos': MAYBE_RELIABLE_YUM_INSTALL, - 'redhat': MAYBE_RELIABLE_YUM_INSTALL, - 'debian': ['apt', 'install', '-y'], - 'ubuntu': ['apt', 'install', '-y'], - 'opensuse': ZYPPER_INSTALL, - 'suse': ZYPPER_INSTALL, + "redhat": MAYBE_RELIABLE_YUM_INSTALL, + "debian": ["apt", "install", "-y"], + "suse": ZYPPER_INSTALL, } - # List of base system packages required to enable ci automation CI_SYSTEM_BASE_PKGS = { - 'common': ['make', 'sudo', 'tar'], - 'eurolinux': ['python3-tox'], - 'miraclelinux': ['python3-tox'], - 'redhat': ['python3-tox'], - 'centos': ['python3-tox'], - 'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'], - 'debian': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild']} + "common": ["make", "sudo", "tar"], + "eurolinux": ["python3-tox"], + "miraclelinux": ["python3-tox"], + "redhat": ["python3-tox"], + "centos": ["python3-tox"], + "ubuntu": ["devscripts", "python3-dev", "libssl-dev", "tox", "sbuild"], + "debian": ["devscripts", "python3-dev", "libssl-dev", "tox", "sbuild"], +} # JSON definition of distro-specific package dependencies @@ -107,36 +110,70 @@ def get_parser(): """Return an argument parser for this command.""" parser = ArgumentParser(description=__doc__) parser.add_argument( - '-r', '--requirements-file', type=str, dest='req_files', - action='append', default=None, - help='pip-style requirements file [default=%s]' % DEFAULT_REQUIREMENTS) + "-r", + "--requirements-file", + type=str, + dest="req_files", + action="append", + default=None, + help="pip-style requirements file [default=%s]" % DEFAULT_REQUIREMENTS, + ) parser.add_argument( - '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(), - help='The name of the distro to generate package deps for.') + "-d", + "--distro", + type=str, + choices=DISTRO_PKG_TYPE_MAP.keys(), + help="The name of the distro to generate package deps for.", + ) deptype = parser.add_mutually_exclusive_group() deptype.add_argument( - '-R', '--runtime-requires', action='store_true', default=False, - dest='runtime_requires', - help='Print only runtime required packages') + "-R", + "--runtime-requires", + action="store_true", + default=False, + dest="runtime_requires", + help="Print only runtime required packages", + ) deptype.add_argument( - '-b', '--build-requires', action='store_true', default=False, - dest='build_requires', help='Print only buildtime required packages') + "-b", + "--build-requires", + action="store_true", + default=False, + dest="build_requires", + help="Print only buildtime required packages", + ) parser.add_argument( - '--dry-run', action='store_true', default=False, dest='dry_run', - help='Dry run the install, making no package changes.') + "--dry-run", + action="store_true", + default=False, + dest="dry_run", + help="Dry run the install, making no package changes.", + ) parser.add_argument( - '-s', '--system-pkg-names', action='store_true', default=False, - dest='system_pkg_names', - help='Generate distribution package names (python3-pkgname).') + "-s", + "--system-pkg-names", + action="store_true", + default=False, + dest="system_pkg_names", + help="Generate distribution package names (python3-pkgname).", + ) parser.add_argument( - '-i', '--install', action='store_true', default=False, - dest='install', - help='When specified, install the required system packages.') + "-i", + "--install", + action="store_true", + default=False, + dest="install", + help="When specified, install the required system packages.", + ) parser.add_argument( - '-t', '--test-distro', action='store_true', default=False, - dest='test_distro', - help='Additionally install continuous integration system packages ' - 'required for build and test automation.') + "-t", + "--test-distro", + action="store_true", + default=False, + dest="test_distro", + help="Additionally install continuous integration system packages " + "required for build and test automation.", + ) return parser @@ -150,7 +187,7 @@ def get_package_deps_from_json(topdir, distro): @return: Dict containing "requires", "build-requires" and "rename" lists for a given distribution. """ - with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream: + with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), "r") as stream: deps = json.loads(stream.read()) if distro is None: return {} @@ -170,11 +207,11 @@ def parse_pip_requirements(requirements_path): continue # remove pip-style markers - dep = line.split(';')[0] + dep = line.split(";")[0] # remove version requirements - if re.search('[>=.<]+', dep): - dep_names.append(re.split(r'[>=.<]+', dep)[0].strip()) + if re.search("[>=.<]+", dep): + dep_names.append(re.split(r"[>=.<]+", dep)[0].strip()) else: dep_names.append(dep) return dep_names @@ -197,16 +234,15 @@ def translate_pip_to_system_pkg(pip_requires, renames): if rename: translated_names.append(rename) else: - translated_names.append( - standard_pkg_name.format(prefix, pip_name)) + translated_names.append(standard_pkg_name.format(prefix, pip_name)) return translated_names def main(distro): parser = get_parser() args = parser.parse_args() - if 'CLOUD_INIT_TOP_D' in os.environ: - topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) + if "CLOUD_INIT_TOP_D" in os.environ: + topd = os.path.realpath(os.environ.get("CLOUD_INIT_TOP_D")) else: topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -215,45 +251,52 @@ def main(distro): if args.req_files: sys.stderr.write( "Parameter --test-distro overrides --requirements-file. Use " - "one or the other.\n") + "one or the other.\n" + ) sys.exit(1) - args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS), - os.path.join(topd, 'test-' + DEFAULT_REQUIREMENTS)] + args.req_files = [ + os.path.join(topd, DEFAULT_REQUIREMENTS), + os.path.join(topd, "test-" + DEFAULT_REQUIREMENTS), + ] args.install = True if args.req_files is None: args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS)] if not os.path.isfile(args.req_files[0]): - sys.stderr.write("Unable to locate '%s' file that should " - "exist in cloud-init root directory." % - args.req_files[0]) + sys.stderr.write( + "Unable to locate '%s' file that should " + "exist in cloud-init root directory." % args.req_files[0] + ) sys.exit(1) bad_files = [r for r in args.req_files if not os.path.isfile(r)] if bad_files: sys.stderr.write( - "Unable to find requirements files: %s\n" % ','.join(bad_files)) + "Unable to find requirements files: %s\n" % ",".join(bad_files) + ) sys.exit(1) pip_pkg_names = set() for req_path in args.req_files: pip_pkg_names.update(set(parse_pip_requirements(req_path))) deps_from_json = get_package_deps_from_json(topd, args.distro) - renames = deps_from_json.get('renames', {}) - translated_pip_names = translate_pip_to_system_pkg( - pip_pkg_names, renames) + renames = deps_from_json.get("renames", {}) + translated_pip_names = translate_pip_to_system_pkg(pip_pkg_names, renames) all_deps = [] select_requires = [args.build_requires, args.runtime_requires] if args.distro: if not any(select_requires): all_deps.extend( - translated_pip_names + deps_from_json['requires'] + - deps_from_json['build-requires']) + translated_pip_names + + deps_from_json["requires"] + + deps_from_json["build-requires"] + ) else: if args.build_requires: - all_deps.extend(deps_from_json['build-requires']) + all_deps.extend(deps_from_json["build-requires"]) else: all_deps.extend( - translated_pip_names + deps_from_json['requires']) + translated_pip_names + deps_from_json["requires"] + ) else: if args.system_pkg_names: all_deps = translated_pip_names @@ -263,33 +306,45 @@ def main(distro): if args.install: pkg_install(all_deps, args.distro, args.test_distro, args.dry_run) else: - print('\n'.join(all_deps)) + print("\n".join(all_deps)) def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" if test_distro: - pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS['common'] + pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS["common"] distro_base_pkgs = CI_SYSTEM_BASE_PKGS.get(distro, []) pkg_list += distro_base_pkgs - print('Installing deps: {0}{1}'.format( - '(dryrun)' if dry_run else '', ' '.join(pkg_list))) + print( + "Installing deps: {0}{1}".format( + "(dryrun)" if dry_run else "", " ".join(pkg_list) + ) + ) install_cmd = [] if dry_run: - install_cmd.append('echo') + install_cmd.append("echo") if os.geteuid() != 0: - install_cmd.append('sudo') + install_cmd.append("sudo") - cmd = DISTRO_INSTALL_PKG_CMD[distro] - if dry_run and distro in DRY_DISTRO_INSTALL_PKG_CMD: - cmd = DRY_DISTRO_INSTALL_PKG_CMD[distro] + distro_family = DISTRO_PKG_TYPE_MAP[distro] + if dry_run and distro_family in DRYRUN_DISTRO_INSTALL_PKG_CMD: + cmd = DRYRUN_DISTRO_INSTALL_PKG_CMD[distro_family] + else: + cmd = DISTRO_INSTALL_PKG_CMD[distro_family] install_cmd.extend(cmd) - if distro in ['centos', 'redhat', 'rocky', 'eurolinux']: + if distro in ["centos", "redhat", "rocky", "eurolinux"]: # CentOS and Redhat need epel-release to access oauthlib and jsonschema - subprocess.check_call(install_cmd + ['epel-release']) - if distro in ['suse', 'opensuse', 'redhat', 'rocky', 'centos', 'eurolinux']: - pkg_list.append('rpm-build') + subprocess.check_call(install_cmd + ["epel-release"]) + if distro in [ + "suse", + "opensuse", + "redhat", + "rocky", + "centos", + "eurolinux", + ]: + pkg_list.append("rpm-build") subprocess.check_call(install_cmd + pkg_list) |