path: root/setup.py

#!/usr/bin/env python3
#
#  Copyright (C) 2016 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
#        Benjamin Schubert <bschubert15@bloomberg.net>

import os
from pathlib import Path
import re
import shutil
import subprocess
import sys

# Add local directory to the path, in order to be able to import versioneer
sys.path.append(os.path.dirname(__file__))
import versioneer  # noqa


##################################################################
# Python requirements
##################################################################
REQUIRED_PYTHON_MAJOR = 3
REQUIRED_PYTHON_MINOR = 5

if sys.version_info[0] != REQUIRED_PYTHON_MAJOR or sys.version_info[1] < REQUIRED_PYTHON_MINOR:
    print("BuildStream requires Python >= 3.5")
    sys.exit(1)

try:
    from setuptools import setup, find_packages, Command, Extension
    from setuptools.command.easy_install import ScriptWriter
    from setuptools.command.test import test as TestCommand
except ImportError:
    print("BuildStream requires setuptools in order to build. Install it using"
          " your package manager (usually python3-setuptools) or via pip (pip3"
          " install setuptools).")
    sys.exit(1)


##################################################################
# Bubblewrap requirements
##################################################################
REQUIRED_BWRAP_MAJOR = 0
REQUIRED_BWRAP_MINOR = 1
REQUIRED_BWRAP_PATCH = 2


def warn_bwrap(reason):
    print(reason +
          "\nBuildStream requires Bubblewrap (bwrap {}.{}.{} or better),"
          " during local builds, for"
          " sandboxing the build environment.\nInstall it using your package manager"
          " (usually bwrap or bubblewrap) otherwise you will be limited to"
          " remote builds only.".format(REQUIRED_BWRAP_MAJOR, REQUIRED_BWRAP_MINOR, REQUIRED_BWRAP_PATCH))


def bwrap_too_old(major, minor, patch):
    if major < REQUIRED_BWRAP_MAJOR:
        return True
    elif major == REQUIRED_BWRAP_MAJOR:
        if minor < REQUIRED_BWRAP_MINOR:
            return True
        elif minor == REQUIRED_BWRAP_MINOR:
            return patch < REQUIRED_BWRAP_PATCH
        else:
            return False
    else:
        return False


def check_for_bwrap():
    platform = sys.platform

    if platform.startswith('linux'):
        sandbox = os.environ.get('BST_FORCE_SANDBOX', "bwrap")
        if sandbox != 'bwrap':
            return
        bwrap_path = shutil.which('bwrap')
        if not bwrap_path:
            warn_bwrap("Bubblewrap not found")
            return

        version_bytes = subprocess.check_output([bwrap_path, "--version"]).split()[1]
        version_string = str(version_bytes, "utf-8")
        major, minor, patch = map(int, version_string.split("."))
        if bwrap_too_old(major, minor, patch):
            warn_bwrap("Bubblewrap too old")


###########################################
# List the pre-built man pages to install #
###########################################
#
# Man pages are automatically generated however it was too difficult
# to integrate with setuptools as a step of the build (FIXME !).
#
# To update the man pages in tree before a release, run:
#
#     tox -e man
#
# Then commit the result.
#
def list_man_pages():
    bst_dir = os.path.dirname(os.path.abspath(__file__))
    man_dir = os.path.join(bst_dir, 'man')
    try:
        man_pages = os.listdir(man_dir)
        return [os.path.join('man', page) for page in man_pages]
    except FileNotFoundError:
        # Do not error out when 'man' directory does not exist
        return []


#####################################################
# List the data files needed by buildstream.testing #
#####################################################
#
# List the datafiles which need to be installed for the
# buildstream.testing package
#
def list_testing_datafiles():
    bst_dir = Path(os.path.dirname(os.path.abspath(__file__)))
    data_dir = bst_dir.joinpath('src', 'buildstream', 'testing', '_sourcetests', 'project')
    return [str(f) for f in data_dir.rglob('*')]


#####################################################
#                Conditional Checks                 #
#####################################################
#
# Because setuptools... there is no way to pass an option to
# the setup.py explicitly at install time.
#
# So screw it, lets just use an env var.
bst_install_entry_points = {
    'console_scripts': [
        'bst-artifact-server = buildstream._cas.casserver:server_main'
    ],
}

if not os.environ.get('BST_ARTIFACTS_ONLY', ''):
    check_for_bwrap()
    bst_install_entry_points['console_scripts'] += [
        'bst = buildstream._frontend:cli'
    ]

#####################################################
#    Monkey-patching setuptools for performance     #
#####################################################
#
# The template of easy_install.ScriptWriter is inefficient in our case as it
# imports pkg_resources. Patching the template only doesn't work because of the
# old string formatting used (%). This forces us to overwrite the class function
# as well.
#
# The patch was inspired from https://github.com/ninjaaron/fast-entry_points
# which we believe was also inspired from the code from `setuptools` project.
TEMPLATE = '''\
# -*- coding: utf-8 -*-
import sys

from {0} import {1}

if __name__ == '__main__':
    sys.exit({2}())'''


# Modify the get_args() function of the ScriptWriter class
# Note: the pylint no-member warning has been disabled as the functions: get_header(),
# ensure_safe_name() and _get_script_args() are all members of this class.
# pylint: disable=no-member
@classmethod
def get_args(cls, dist, header=None):
    if header is None:
        header = cls.get_header()
    for name, ep in dist.get_entry_map('console_scripts').items():
        cls._ensure_safe_name(name)
        script_text = TEMPLATE.format(ep.module_name, ep.attrs[0], '.'.join(ep.attrs))
        args = cls._get_script_args('console', name, header, script_text)
        for res in args:
            yield res


ScriptWriter.get_args = get_args


#####################################################
#         gRPC command for code generation          #
#####################################################
class BuildGRPC(Command):
    """Command to generate project *_pb2.py modules from proto files."""

    description = 'build gRPC protobuf modules'
    user_options = []

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        try:
            import grpc_tools.command
        except ImportError:
            print("BuildStream requires grpc_tools in order to build gRPC modules.\n"
                  "Install it via pip (pip3 install grpcio-tools).")
            exit(1)

        protos_root = 'src/buildstream/_protos'

        grpc_tools.command.build_package_protos(protos_root)

        # Postprocess imports in generated code
        for root, _, files in os.walk(protos_root):
            for filename in files:
                if filename.endswith('.py'):
                    path = os.path.join(root, filename)
                    with open(path, 'r') as f:
                        code = f.read()

                    # All protos are in buildstream._protos
                    code = re.sub(r'^from ', r'from buildstream._protos.',
                                  code, flags=re.MULTILINE)
                    # Except for the core google.protobuf protos
                    code = re.sub(r'^from buildstream._protos.google.protobuf', r'from google.protobuf',
                                  code, flags=re.MULTILINE)

                    with open(path, 'w') as f:
                        f.write(code)


#####################################################
#                   Pytest command                  #
#####################################################
class PyTest(TestCommand):
    """Defines a pytest command class to run tests from setup.py"""

    user_options = TestCommand.user_options + [
        ("addopts=", None, "Arguments to pass to pytest"),
        ('index-url=', None, "Specify an index url from which to retrieve "
                             "dependencies"),
    ]

    # pylint: disable=attribute-defined-outside-init
    def initialize_options(self):
        super().initialize_options()
        self.addopts = ""
        self.index_url = None

    def run(self):
        if self.index_url is not None:
            if self.distribution.command_options.get("easy_install") is None:
                self.distribution.command_options["easy_install"] = {}

            self.distribution.command_options["easy_install"]["index_url"] = (
                "cmdline", self.index_url,
            )
        super().run()

    def run_tests(self):
        import shlex
        import pytest

        errno = pytest.main(shlex.split(self.addopts))

        if errno:
            raise SystemExit(errno)


def get_cmdclass():
    cmdclass = {
        'build_grpc': BuildGRPC,
        'pytest': PyTest,
    }
    cmdclass.update(versioneer.get_cmdclass())
    return cmdclass


#####################################################
#               Gather requirements                 #
#####################################################
with open('requirements/dev-requirements.in') as dev_reqs:
    dev_requires = dev_reqs.read().splitlines()

with open('requirements/requirements.in') as install_reqs:
    install_requires = install_reqs.read().splitlines()

#####################################################
#     Prepare package description from README       #
#####################################################
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                       'README.rst')) as readme:
    long_description = readme.read()


#####################################################
#            Setup Cython and extensions            #
#####################################################
# We want to ensure that source distributions always
# include the .c files, in order to allow users to
# not need cython when building.
def assert_cython_required():
    if "sdist" not in sys.argv:
        return

    print("Cython is required when building 'sdist' in order to "
          "ensure source distributions can be built without Cython. "
          "Please install it using your package manager (usually 'python3-cython') "
          "or pip (pip install cython).",
          file=sys.stderr)

    raise SystemExit(1)


try:
    ENABLE_CYTHON_TRACE = int(os.environ.get("BST_CYTHON_TRACE", "0"))
except ValueError:
    print("BST_CYTHON_TRACE must be an integer. Please set it to '1' to enable, '0' to disable", file=sys.stderr)
    raise SystemExit(1)


extension_macros = [
    ("CYTHON_TRACE", ENABLE_CYTHON_TRACE)
]


def cythonize(extensions, **kwargs):
    try:
        from Cython.Build import cythonize as _cythonize
    except ImportError:
        assert_cython_required()

        print("Cython not found. Using preprocessed c files instead")

        missing_c_sources = []

        for extension in extensions:
            for source in extension.sources:
                if source.endswith(".pyx"):
                    c_file = source.replace(".pyx", ".c")

                    if not os.path.exists(c_file):
                        missing_c_sources.append((extension, c_file))

        if missing_c_sources:
            for extension, source in missing_c_sources:
                print("Missing '{}' for building extension '{}'".format(source, extension.name))

            raise SystemExit(1)
        return extensions

    return _cythonize(extensions, **kwargs)


def register_cython_module(module_name, dependencies=None):
    def files_from_module(modname):
        basename = "src/{}".format(modname.replace(".", "/"))
        return "{}.pyx".format(basename), "{}.pxd".format(basename)

    if dependencies is None:
        dependencies = []

    implementation_file, definition_file = files_from_module(module_name)

    assert os.path.exists(implementation_file)

    depends = []
    if os.path.exists(definition_file):
        depends.append(definition_file)

    for module in dependencies:
        imp_file, def_file = files_from_module(module)
        assert os.path.exists(imp_file), "Dependency file not found: {}".format(imp_file)
        assert os.path.exists(def_file), "Dependency declaration file not found: {}".format(def_file)

        depends.append(imp_file)
        depends.append(def_file)

    BUILD_EXTENSIONS.append(Extension(
        name=module_name,
        sources=[implementation_file],
        depends=depends,
        define_macros=extension_macros,
    ))


BUILD_EXTENSIONS = []

register_cython_module("buildstream.node")
register_cython_module("buildstream._loader._loader")
register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"])
register_cython_module("buildstream._yaml", dependencies=["buildstream.node"])
register_cython_module("buildstream._variables", dependencies=["buildstream.node"])

#####################################################
#             Main setup() Invocation               #
#####################################################
setup(name='BuildStream',
      # Use versioneer
      version=versioneer.get_version(),
      cmdclass=get_cmdclass(),

      author='BuildStream Developers',
      author_email='buildstream-list@gnome.org',
      classifiers=[
          'Environment :: Console',
          'Intended Audience :: Developers',
          'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
          'Operating System :: POSIX',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.5',
          'Programming Language :: Python :: 3.6',
          'Programming Language :: Python :: 3.7',
          'Topic :: Software Development :: Build Tools'
      ],
      description='A framework for modelling build pipelines in YAML',
      license='LGPL',
      long_description=long_description,
      long_description_content_type='text/x-rst; charset=UTF-8',
      url='https://buildstream.build',
      project_urls={
          'Source': 'https://gitlab.com/BuildStream/buildstream',
          'Documentation': 'https://docs.buildstream.build',
          'Tracker': 'https://gitlab.com/BuildStream/buildstream/issues',
          'Mailing List': 'https://mail.gnome.org/mailman/listinfo/buildstream-list'
      },
      python_requires='~={}.{}'.format(REQUIRED_PYTHON_MAJOR, REQUIRED_PYTHON_MINOR),
      package_dir={'': 'src'},
      packages=find_packages(where='src', exclude=('tests', 'tests.*')),
      package_data={'buildstream': ['plugins/*/*.py', 'plugins/*/*.yaml',
                                    'data/*.yaml', 'data/*.sh.in',
                                    *list_testing_datafiles()]},
      data_files=[
          # This is a weak attempt to integrate with the user nicely,
          # installing things outside of the python package itself with pip is
          # not recommended, but there seems to be no standard structure for
          # addressing this; so just installing this here.
          #
          # These do not get installed in developer mode (`pip install --user -e .`)
          #
          # The completions are ignored by bash unless it happens to be installed
          # in the right directory; this is more like a weak statement that we
          # attempt to install bash completion scriptlet.
          #
          ('share/man/man1', list_man_pages()),
          ('share/bash-completion/completions', [
              os.path.join('src', 'buildstream', 'data', 'bst')
          ])
      ],
      install_requires=install_requires,
      entry_points=bst_install_entry_points,
      tests_require=dev_requires,
      ext_modules=cythonize(
          BUILD_EXTENSIONS,
          compiler_directives={
              # Version of python to use
              # https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#arguments
              "language_level": "3",
              # Enable line tracing when requested only, this is needed in order to generate coverage.
              "linetrace": bool(ENABLE_CYTHON_TRACE),
              "profile": os.environ.get("BST_CYTHON_PROFILE", False),
          }
      ),
      zip_safe=False)