diff options
Diffstat (limited to 'src')
110 files changed, 363 insertions, 360 deletions
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index cf82a1636..28dfaa4b4 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -30,12 +30,11 @@ artifact composite interaction away from Element class import os +from . import _yaml, utils from ._exceptions import ArtifactError from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto -from . import _yaml -from . import utils -from .types import Scope from .storage._casbaseddirectory import CasBasedDirectory +from .types import Scope # An Artifact class to abstract artifact operations diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 46fc5fbfb..b7bd824d8 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -19,15 +19,15 @@ # Tristan Maat <tristan.maat@codethink.co.uk> import os + import grpc +from . import utils from ._basecache import BaseCache from ._cas.casremote import BlobNotFound -from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError, RemoteError -from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc - +from ._exceptions import ArtifactError, CacheError, CASError, CASRemoteError, RemoteError +from ._protos.buildstream.v2 import artifact_pb2, artifact_pb2_grpc, buildstream_pb2, buildstream_pb2_grpc from ._remote import BaseRemote -from . import utils # ArtifactRemote(): diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py index 4066cef06..ad42cba3a 100644 --- a/src/buildstream/_artifactelement.py +++ b/src/buildstream/_artifactelement.py @@ -19,8 +19,7 @@ from typing import TYPE_CHECKING -from . import Element -from . import _cachekey +from . import Element, _cachekey from ._exceptions import ArtifactElementError from ._loader.metaelement import MetaElement from .types import Scope diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index 91eae6ae4..d4166dffd 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -21,14 +21,12 @@ from fnmatch import fnmatch from itertools import chain from typing import TYPE_CHECKING -from . import utils -from . import _yaml +from . import _yaml, utils from ._cas import CASRemote +from ._exceptions import CacheError, LoadError, RemoteError from ._message import Message, MessageType -from ._exceptions import LoadError, RemoteError, CacheError from ._remote import RemoteSpec, RemoteType - if TYPE_CHECKING: from typing import Optional, Type from ._exceptions import BstError diff --git a/src/buildstream/_cachekey.py b/src/buildstream/_cachekey.py index dd9207516..5bf11e5dc 100644 --- a/src/buildstream/_cachekey.py +++ b/src/buildstream/_cachekey.py @@ -22,7 +22,6 @@ import hashlib import ujson - # Internal record of the size of a cache key _CACHEKEY_SIZE = len(hashlib.sha256().hexdigest()) diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 7936121ea..a5757f175 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -18,26 +18,24 @@ # Authors: # Jürg Billeter <juerg.billeter@codethink.co.uk> -import itertools -import os -import stat import contextlib import ctypes +import itertools import multiprocessing +import os import signal +import stat import time -from typing import Optional, List +from typing import List, Optional import grpc -from .._protos.google.rpc import code_pb2 +from .. import _signals, utils +from .._exceptions import CASCacheError from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.build.buildgrid import local_cas_pb2 - -from .. import _signals, utils +from .._protos.google.rpc import code_pb2 from ..types import FastEnum, SourceRef -from .._exceptions import CASCacheError - from .casdprocessmanager import CASDProcessManager from .casremote import _CASBatchRead, _CASBatchUpdate diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py index 637c4e0b1..55121b4b3 100644 --- a/src/buildstream/_cas/casdprocessmanager.py +++ b/src/buildstream/_cas/casdprocessmanager.py @@ -25,17 +25,16 @@ import stat import subprocess import tempfile import time -import psutil import grpc - -from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc -from .._protos.build.buildgrid import local_cas_pb2_grpc -from .._protos.google.bytestream import bytestream_pb2_grpc +import psutil from .. import _signals, utils from .._exceptions import CASCacheError from .._message import Message, MessageType +from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc +from .._protos.build.buildgrid import local_cas_pb2_grpc +from .._protos.google.bytestream import bytestream_pb2_grpc _CASD_MAX_LOGFILES = 10 _CASD_TIMEOUT = 300 # in seconds diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py index 93f4e500c..16a5d8de5 100644 --- a/src/buildstream/_cas/casremote.py +++ b/src/buildstream/_cas/casremote.py @@ -15,11 +15,10 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. # -from .._protos.google.rpc import code_pb2 +from .._exceptions import CASRemoteError from .._protos.build.buildgrid import local_cas_pb2 - +from .._protos.google.rpc import code_pb2 from .._remote import BaseRemote -from .._exceptions import CASRemoteError # The default limit for gRPC messages is 4 MiB. # Limit payload to 1 MiB to leave sufficient headroom for metadata. diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py index 71d7d9071..c90a98cac 100644 --- a/src/buildstream/_cas/casserver.py +++ b/src/buildstream/_cas/casserver.py @@ -17,33 +17,29 @@ # Authors: # Jürg Billeter <juerg.billeter@codethink.co.uk> -from concurrent import futures -from enum import Enum import contextlib import logging import os import signal import sys +from concurrent import futures +from enum import Enum +import click import grpc from google.protobuf.message import DecodeError -import click -from .._protos.build.bazel.remote.execution.v2 import ( - remote_execution_pb2, - remote_execution_pb2_grpc, -) -from .._protos.google.bytestream import bytestream_pb2_grpc +from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc from .._protos.build.buildgrid import local_cas_pb2 from .._protos.buildstream.v2 import ( - buildstream_pb2, - buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc, + buildstream_pb2, + buildstream_pb2_grpc, source_pb2, source_pb2_grpc, ) - +from .._protos.google.bytestream import bytestream_pb2_grpc # Note: We'd ideally like to avoid imports from the core codebase as # much as possible, since we're expecting to eventually split this # module off into its own project. @@ -51,10 +47,9 @@ from .._protos.buildstream.v2 import ( # Not enough that we'd like to duplicate code, but enough that we want # to make it very obvious what we're using, so in this case we import # the specific methods we'll be using. -from ..utils import save_file_atomic, _remove_path_with_parents +from ..utils import _remove_path_with_parents, save_file_atomic from .casdprocessmanager import CASDProcessManager - # The default limit for gRPC messages is 4 MiB. # Limit payload to 1 MiB to leave sufficient headroom for metadata. _MAX_PAYLOAD_BYTES = 1024 * 1024 diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index c0e92b98e..6f7629e9f 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -19,21 +19,20 @@ import os import shutil -from . import utils -from . import _site -from . import _yaml + +from . import _site, _yaml, utils +from ._artifactcache import ArtifactCache +from ._cas import CASCache, CASLogLevel from ._exceptions import LoadError -from .exceptions import LoadErrorReason from ._messenger import Messenger -from ._profile import Topics, PROFILER from ._platform import Platform -from ._artifactcache import ArtifactCache +from ._profile import PROFILER, Topics from ._sourcecache import SourceCache -from ._cas import CASCache, CASLogLevel -from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction -from ._workspaces import Workspaces, WorkspaceProjectCache +from ._workspaces import WorkspaceProjectCache, Workspaces +from .exceptions import LoadErrorReason from .node import Node from .sandbox import SandboxRemote +from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction # Context() diff --git a/src/buildstream/_frontend/__init__.py b/src/buildstream/_frontend/__init__.py index febd4979d..0744b818e 100644 --- a/src/buildstream/_frontend/__init__.py +++ b/src/buildstream/_frontend/__init__.py @@ -17,6 +17,7 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> import os + from .cli import cli if "_BST_COMPLETION" not in os.environ: diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index b77cfc130..df840dec5 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -17,30 +17,27 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -from contextlib import contextmanager +import datetime import os import sys import traceback -import datetime +from contextlib import contextmanager from textwrap import TextWrapper + import click from click import UsageError # Import buildstream public symbols -from .. import Scope - +from .. import Scope, node, utils # Import various buildstream internals from .._context import Context -from .._project import Project -from .._exceptions import BstError, StreamError, LoadError, AppError -from ..exceptions import LoadErrorReason +from .._exceptions import AppError, BstError, LoadError, StreamError from .._message import Message, MessageType, unconditional_messages +from .._project import Project from .._stream import Stream +from ..exceptions import LoadErrorReason from ..types import _SchedulerErrorAction -from .. import node -from .. import utils from ..utils import UtilError - # Import frontend assets from .profile import Profile from .status import Status diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index d5fa47091..26dc6ec7c 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1,15 +1,15 @@ import os +import shutil import sys from functools import partial -import shutil import click + from .. import _yaml -from .._exceptions import BstError, LoadError, AppError -from .complete import main_bashcomplete, complete_path, CompleteUnhandled -from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection +from .._exceptions import AppError, BstError, LoadError +from ..types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction from ..utils import UtilError - +from .complete import CompleteUnhandled, complete_path, main_bashcomplete ################################################################## # Helper classes and methods for Click # diff --git a/src/buildstream/_frontend/complete.py b/src/buildstream/_frontend/complete.py index 45e857e3e..4afd75c92 100644 --- a/src/buildstream/_frontend/complete.py +++ b/src/buildstream/_frontend/complete.py @@ -36,7 +36,7 @@ import copy import os import click -from click.core import MultiCommand, Option, Argument +from click.core import Argument, MultiCommand, Option from click.parser import split_arg_string WORDBREAK = "=" diff --git a/src/buildstream/_frontend/linuxapp.py b/src/buildstream/_frontend/linuxapp.py index 987b023bd..2cb8e2ec4 100644 --- a/src/buildstream/_frontend/linuxapp.py +++ b/src/buildstream/_frontend/linuxapp.py @@ -17,6 +17,7 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> import os + import click from .app import App diff --git a/src/buildstream/_frontend/profile.py b/src/buildstream/_frontend/profile.py index 02b601c66..b5e41cc8b 100644 --- a/src/buildstream/_frontend/profile.py +++ b/src/buildstream/_frontend/profile.py @@ -16,8 +16,9 @@ # # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -import re import copy +import re + import click diff --git a/src/buildstream/_frontend/status.py b/src/buildstream/_frontend/status.py index a3f0d8aa7..953f72bb7 100644 --- a/src/buildstream/_frontend/status.py +++ b/src/buildstream/_frontend/status.py @@ -16,10 +16,11 @@ # # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +import curses import os import sys -import curses from collections import OrderedDict + import click # Import a widget internal for formatting time codes diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index f07e3dba0..2f18aeb50 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -18,20 +18,21 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> import datetime import os -from collections import defaultdict, OrderedDict -from contextlib import ExitStack -from mmap import mmap import re import textwrap -from ruamel import yaml +from collections import OrderedDict, defaultdict +from contextlib import ExitStack +from mmap import mmap + import click +from ruamel import yaml -from .profile import Profile from .. import Scope from .. import __version__ as bst_version from .._exceptions import ImplError from .._message import MessageType from ..storage.directory import _FileType +from .profile import Profile # These messages are printed a bit differently ERROR_MESSAGES = [MessageType.FAIL, MessageType.ERROR, MessageType.BUG] diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py index e3b246677..6f4c04d27 100644 --- a/src/buildstream/_gitsourcebase.py +++ b/src/buildstream/_gitsourcebase.py @@ -25,16 +25,14 @@ import os import re import shutil +from configparser import RawConfigParser from io import StringIO from tempfile import TemporaryFile -from configparser import RawConfigParser - -from .source import Source, SourceError, SourceFetcher -from .types import CoreWarnings from . import utils -from .types import FastEnum -from .utils import move_atomic, DirectoryExistsError +from .source import Source, SourceError, SourceFetcher +from .types import CoreWarnings, FastEnum +from .utils import DirectoryExistsError, move_atomic GIT_MODULES = ".gitmodules" EXACT_TAG_PATTERN = r"(?P<tag>.*)-0-g(?P<commit>.*)" diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py index 1556bc613..24f7ed80a 100644 --- a/src/buildstream/_includes.py +++ b/src/buildstream/_includes.py @@ -1,8 +1,9 @@ import os + from . import _yaml -from .node import MappingNode, ScalarNode, SequenceNode from ._exceptions import LoadError from .exceptions import LoadErrorReason +from .node import MappingNode, ScalarNode, SequenceNode # Includes() diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py index fd5cac2ae..7980e776b 100644 --- a/src/buildstream/_loader/__init__.py +++ b/src/buildstream/_loader/__init__.py @@ -17,7 +17,7 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -from .metasource import MetaSource -from .metaelement import MetaElement from .loadcontext import LoadContext from .loader import Loader +from .metaelement import MetaElement +from .metasource import MetaSource diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 81e5fb032..019aed1f9 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -20,22 +20,21 @@ import os from contextlib import suppress -from .._exceptions import LoadError -from ..exceptions import LoadErrorReason from .. import _yaml +from .._exceptions import LoadError +from .._includes import Includes +from .._message import Message, MessageType +from .._profile import PROFILER, Topics from ..element import Element +from ..exceptions import LoadErrorReason from ..node import Node -from .._profile import Topics, PROFILER -from .._includes import Includes - -from ._loader import valid_chars_name -from .types import Symbol, extract_depends_from_node +from ..types import CoreWarnings, _KeyStrength from . import loadelement +from ._loader import valid_chars_name from .loadelement import Dependency, LoadElement from .metaelement import MetaElement from .metasource import MetaSource -from ..types import CoreWarnings, _KeyStrength -from .._message import Message, MessageType +from .types import Symbol, extract_depends_from_node # Loader(): diff --git a/src/buildstream/_messenger.py b/src/buildstream/_messenger.py index 3a32a2467..24c99732c 100644 --- a/src/buildstream/_messenger.py +++ b/src/buildstream/_messenger.py @@ -17,16 +17,14 @@ # Authors: # Angelos Evripiotis <jevripiotis@bloomberg.net> -import os import datetime +import os from contextlib import contextmanager -from . import _signals -from . import utils +from . import _signals, utils from ._exceptions import BstError from ._message import Message, MessageType - _RENDER_INTERVAL = datetime.timedelta(seconds=1) diff --git a/src/buildstream/_options/optionarch.py b/src/buildstream/_options/optionarch.py index 651afbea5..e3d13550b 100644 --- a/src/buildstream/_options/optionarch.py +++ b/src/buildstream/_options/optionarch.py @@ -18,8 +18,8 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> from .._exceptions import LoadError, PlatformError -from ..exceptions import LoadErrorReason from .._platform import Platform +from ..exceptions import LoadErrorReason from .optionenum import OptionEnum diff --git a/src/buildstream/_options/optionbool.py b/src/buildstream/_options/optionbool.py index 57dbd160e..97e8d8e62 100644 --- a/src/buildstream/_options/optionbool.py +++ b/src/buildstream/_options/optionbool.py @@ -19,7 +19,7 @@ from .._exceptions import LoadError from ..exceptions import LoadErrorReason -from .option import Option, OPTION_SYMBOLS +from .option import OPTION_SYMBOLS, Option # OptionBool diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py index da3a7dc7e..24d85cef7 100644 --- a/src/buildstream/_options/optionenum.py +++ b/src/buildstream/_options/optionenum.py @@ -19,7 +19,7 @@ from .._exceptions import LoadError from ..exceptions import LoadErrorReason -from .option import Option, OPTION_SYMBOLS +from .option import OPTION_SYMBOLS, Option # OptionEnum diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py index 042ef644c..835c1ef45 100644 --- a/src/buildstream/_options/optionflags.py +++ b/src/buildstream/_options/optionflags.py @@ -19,7 +19,7 @@ from .._exceptions import LoadError from ..exceptions import LoadErrorReason -from .option import Option, OPTION_SYMBOLS +from .option import OPTION_SYMBOLS, Option # OptionFlags diff --git a/src/buildstream/_options/optionos.py b/src/buildstream/_options/optionos.py index 3f4e902c9..cc1cff2cd 100644 --- a/src/buildstream/_options/optionos.py +++ b/src/buildstream/_options/optionos.py @@ -18,6 +18,7 @@ # Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk> import platform + from .optionenum import OptionEnum diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py index b30d1f0ce..00c9d34b3 100644 --- a/src/buildstream/_options/optionpool.py +++ b/src/buildstream/_options/optionpool.py @@ -24,14 +24,13 @@ from .._exceptions import LoadError from ..exceptions import LoadErrorReason from ..node import MappingNode, SequenceNode, _assert_symbol_name from ..types import FastEnum +from .optionarch import OptionArch from .optionbool import OptionBool +from .optioneltmask import OptionEltMask from .optionenum import OptionEnum from .optionflags import OptionFlags -from .optioneltmask import OptionEltMask -from .optionarch import OptionArch from .optionos import OptionOS - _OPTION_TYPES = { OptionBool.OPTION_TYPE: OptionBool, OptionEnum.OPTION_TYPE: OptionEnum, diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index 9edc6f51b..5afced0f5 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -19,17 +19,17 @@ # Jürg Billeter <juerg.billeter@codethink.co.uk> # Tristan Maat <tristan.maat@codethink.co.uk> -import os import itertools -from operator import itemgetter +import os from collections import OrderedDict +from operator import itemgetter from pyroaring import BitMap # pylint: disable=no-name-in-module +from . import Scope from ._exceptions import PipelineError from ._message import Message, MessageType -from ._profile import Topics, PROFILER -from . import Scope +from ._profile import PROFILER, Topics from ._project import ProjectRefStorage from .types import _PipelineSelection diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py index 50a1d8e6c..29b44553e 100644 --- a/src/buildstream/_platform/platform.py +++ b/src/buildstream/_platform/platform.py @@ -23,9 +23,9 @@ import platform import psutil -from .._exceptions import PlatformError, ImplError, SandboxError -from ..sandbox import SandboxDummy from .. import utils +from .._exceptions import ImplError, PlatformError, SandboxError +from ..sandbox import SandboxDummy class Platform: diff --git a/src/buildstream/_pluginfactory/__init__.py b/src/buildstream/_pluginfactory/__init__.py index cd4172392..9af2e1d66 100644 --- a/src/buildstream/_pluginfactory/__init__.py +++ b/src/buildstream/_pluginfactory/__init__.py @@ -15,12 +15,12 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. # +from .elementfactory import ElementFactory from .pluginorigin import PluginOrigin, PluginOriginType, PluginType +from .pluginoriginjunction import PluginOriginJunction from .pluginoriginlocal import PluginOriginLocal from .pluginoriginpip import PluginOriginPip -from .pluginoriginjunction import PluginOriginJunction from .sourcefactory import SourceFactory -from .elementfactory import ElementFactory # load_plugin_origin() diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py index df950abee..b7171c4be 100644 --- a/src/buildstream/_pluginfactory/pluginfactory.py +++ b/src/buildstream/_pluginfactory/pluginfactory.py @@ -18,20 +18,19 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> import os -from typing import Tuple, Type, Iterator +from typing import Iterator, Tuple, Type + from pluginbase import PluginSource -from .. import utils -from .. import _site -from ..plugin import Plugin -from ..source import Source +from .. import _site, utils +from .._exceptions import PluginError +from .._message import Message, MessageType +from .._messenger import Messenger from ..element import Element from ..node import ProvenanceInformation +from ..plugin import Plugin +from ..source import Source from ..utils import UtilError -from .._exceptions import PluginError -from .._messenger import Messenger -from .._message import Message, MessageType - from .pluginorigin import PluginOrigin, PluginType diff --git a/src/buildstream/_pluginfactory/pluginorigin.py b/src/buildstream/_pluginfactory/pluginorigin.py index bd987171d..40c4ab59a 100644 --- a/src/buildstream/_pluginfactory/pluginorigin.py +++ b/src/buildstream/_pluginfactory/pluginorigin.py @@ -15,10 +15,10 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. # -from ..types import FastEnum -from ..node import ScalarNode, MappingNode from .._exceptions import LoadError from ..exceptions import LoadErrorReason +from ..node import MappingNode, ScalarNode +from ..types import FastEnum # PluginType() diff --git a/src/buildstream/_pluginfactory/pluginoriginjunction.py b/src/buildstream/_pluginfactory/pluginoriginjunction.py index 4e0a53cfb..1ed69cff6 100644 --- a/src/buildstream/_pluginfactory/pluginoriginjunction.py +++ b/src/buildstream/_pluginfactory/pluginoriginjunction.py @@ -15,8 +15,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. # from .._exceptions import PluginError - -from .pluginorigin import PluginType, PluginOrigin, PluginOriginType +from .pluginorigin import PluginOrigin, PluginOriginType, PluginType # PluginOriginJunction diff --git a/src/buildstream/_pluginfactory/pluginoriginpip.py b/src/buildstream/_pluginfactory/pluginoriginpip.py index 3a9c63f7e..650cf18ed 100644 --- a/src/buildstream/_pluginfactory/pluginoriginpip.py +++ b/src/buildstream/_pluginfactory/pluginoriginpip.py @@ -17,8 +17,7 @@ import os from .._exceptions import PluginError - -from .pluginorigin import PluginType, PluginOrigin, PluginOriginType +from .pluginorigin import PluginOrigin, PluginOriginType, PluginType # PluginOriginPip diff --git a/src/buildstream/_pluginfactory/sourcefactory.py b/src/buildstream/_pluginfactory/sourcefactory.py index 2ed78f838..ad08e25e7 100644 --- a/src/buildstream/_pluginfactory/sourcefactory.py +++ b/src/buildstream/_pluginfactory/sourcefactory.py @@ -20,6 +20,7 @@ from .pluginfactory import PluginFactory from .pluginorigin import PluginType + # A SourceFactory creates Source instances # in the context of a given factory # diff --git a/src/buildstream/_profile.py b/src/buildstream/_profile.py index 0219e8374..55d125030 100644 --- a/src/buildstream/_profile.py +++ b/src/buildstream/_profile.py @@ -23,10 +23,11 @@ import contextlib import cProfile -import pstats -import os import datetime +import os +import pstats import time + from ._exceptions import ProfileError diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 21dc2b9d2..fb688c266 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -22,29 +22,28 @@ import os import sys from collections import OrderedDict from pathlib import Path + from pluginbase import PluginBase -from . import utils -from . import _site -from . import _yaml -from .utils import UtilError + +from . import _site, _yaml, utils +from ._artifactcache import ArtifactCache from ._artifactelement import ArtifactElement -from ._profile import Topics, PROFILER from ._exceptions import LoadError -from .exceptions import LoadErrorReason +from ._includes import Includes +from ._loader import LoadContext, Loader +from ._message import Message, MessageType from ._options import OptionPool -from ._artifactcache import ArtifactCache +from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin +from ._profile import PROFILER, Topics +from ._projectrefs import ProjectRefs, ProjectRefStorage from ._sourcecache import SourceCache +from ._workspaces import WORKSPACE_PROJECT_FILE +from .element import Element +from .exceptions import LoadErrorReason from .node import ScalarNode, SequenceNode, _assert_symbol_name from .sandbox import SandboxRemote -from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin from .types import CoreWarnings -from ._projectrefs import ProjectRefs, ProjectRefStorage -from ._loader import Loader, LoadContext -from .element import Element -from ._message import Message, MessageType -from ._includes import Includes -from ._workspaces import WORKSPACE_PROJECT_FILE - +from .utils import UtilError # Project Configuration file _PROJECT_CONF_FILE = "project.conf" diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py index 6af470808..e16de28e8 100644 --- a/src/buildstream/_projectrefs.py +++ b/src/buildstream/_projectrefs.py @@ -19,9 +19,9 @@ import os from . import _yaml -from .node import _new_synthetic_file from ._exceptions import LoadError from .exceptions import LoadErrorReason +from .node import _new_synthetic_file # ProjectRefStorage() diff --git a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py index 8c34ef865..e5955e0f1 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py +++ b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py @@ -6,13 +6,17 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + +from buildstream._protos.build.bazel.remote.execution.v2 import ( + remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2, +) +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 -from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/buildstream/v2/buildstream_pb2.py b/src/buildstream/_protos/buildstream/v2/buildstream_pb2.py index 0780c9fa2..7adce4e88 100644 --- a/src/buildstream/_protos/buildstream/v2/buildstream_pb2.py +++ b/src/buildstream/_protos/buildstream/v2/buildstream_pb2.py @@ -6,13 +6,17 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + +from buildstream._protos.build.bazel.remote.execution.v2 import ( + remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2, +) +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 -from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/buildstream/v2/source_pb2.py b/src/buildstream/_protos/buildstream/v2/source_pb2.py index 136c4cb2e..e9ade6449 100644 --- a/src/buildstream/_protos/buildstream/v2/source_pb2.py +++ b/src/buildstream/_protos/buildstream/v2/source_pb2.py @@ -6,13 +6,17 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + +from buildstream._protos.build.bazel.remote.execution.v2 import ( + remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2, +) +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 -from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/google/api/annotations_pb2.py b/src/buildstream/_protos/google/api/annotations_pb2.py index 35585c753..3aaa130d5 100644 --- a/src/buildstream/_protos/google/api/annotations_pb2.py +++ b/src/buildstream/_protos/google/api/annotations_pb2.py @@ -3,16 +3,18 @@ # source: google/api/annotations.proto from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + +from buildstream._protos.google.api import http_pb2 as google_dot_api_dot_http__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.google.api import http_pb2 as google_dot_api_dot_http__pb2 -from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/google/api/annotations_pb2_grpc.py b/src/buildstream/_protos/google/api/annotations_pb2_grpc.py index a89435267..07cb78fe0 100644 --- a/src/buildstream/_protos/google/api/annotations_pb2_grpc.py +++ b/src/buildstream/_protos/google/api/annotations_pb2_grpc.py @@ -1,3 +1,2 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc - diff --git a/src/buildstream/_protos/google/api/http_pb2.py b/src/buildstream/_protos/google/api/http_pb2.py index cf4ce322e..df98dd880 100644 --- a/src/buildstream/_protos/google/api/http_pb2.py +++ b/src/buildstream/_protos/google/api/http_pb2.py @@ -6,6 +6,7 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/src/buildstream/_protos/google/api/http_pb2_grpc.py b/src/buildstream/_protos/google/api/http_pb2_grpc.py index a89435267..07cb78fe0 100644 --- a/src/buildstream/_protos/google/api/http_pb2_grpc.py +++ b/src/buildstream/_protos/google/api/http_pb2_grpc.py @@ -1,3 +1,2 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc - diff --git a/src/buildstream/_protos/google/bytestream/bytestream_pb2.py b/src/buildstream/_protos/google/bytestream/bytestream_pb2.py index d8627d4e2..893703f92 100644 --- a/src/buildstream/_protos/google/bytestream/bytestream_pb2.py +++ b/src/buildstream/_protos/google/bytestream/bytestream_pb2.py @@ -6,13 +6,15 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 + +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/google/longrunning/operations_pb2.py b/src/buildstream/_protos/google/longrunning/operations_pb2.py index 7798de35b..46faf03ba 100644 --- a/src/buildstream/_protos/google/longrunning/operations_pb2.py +++ b/src/buildstream/_protos/google/longrunning/operations_pb2.py @@ -2,19 +2,21 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/longrunning/operations.proto +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 from google.protobuf import descriptor as _descriptor +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 -from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/google/longrunning/operations_pb2_grpc.py b/src/buildstream/_protos/google/longrunning/operations_pb2_grpc.py index 8f89862e7..c9f43638a 100644 --- a/src/buildstream/_protos/google/longrunning/operations_pb2_grpc.py +++ b/src/buildstream/_protos/google/longrunning/operations_pb2_grpc.py @@ -1,8 +1,8 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 from buildstream._protos.google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 class OperationsStub(object): diff --git a/src/buildstream/_protos/google/rpc/code_pb2.py b/src/buildstream/_protos/google/rpc/code_pb2.py index 85a6b2dcd..1c33cd305 100644 --- a/src/buildstream/_protos/google/rpc/code_pb2.py +++ b/src/buildstream/_protos/google/rpc/code_pb2.py @@ -2,11 +2,12 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/rpc/code.proto -from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/src/buildstream/_protos/google/rpc/code_pb2_grpc.py b/src/buildstream/_protos/google/rpc/code_pb2_grpc.py index a89435267..07cb78fe0 100644 --- a/src/buildstream/_protos/google/rpc/code_pb2_grpc.py +++ b/src/buildstream/_protos/google/rpc/code_pb2_grpc.py @@ -1,3 +1,2 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc - diff --git a/src/buildstream/_protos/google/rpc/status_pb2.py b/src/buildstream/_protos/google/rpc/status_pb2.py index ae0be7d02..04d8a8145 100644 --- a/src/buildstream/_protos/google/rpc/status_pb2.py +++ b/src/buildstream/_protos/google/rpc/status_pb2.py @@ -2,16 +2,17 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/rpc/status.proto +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/src/buildstream/_protos/google/rpc/status_pb2_grpc.py b/src/buildstream/_protos/google/rpc/status_pb2_grpc.py index a89435267..07cb78fe0 100644 --- a/src/buildstream/_protos/google/rpc/status_pb2_grpc.py +++ b/src/buildstream/_protos/google/rpc/status_pb2_grpc.py @@ -1,3 +1,2 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc - diff --git a/src/buildstream/_remote.py b/src/buildstream/_remote.py index d8b8e68fe..9fafc1414 100644 --- a/src/buildstream/_remote.py +++ b/src/buildstream/_remote.py @@ -21,7 +21,7 @@ from urllib.parse import urlparse import grpc -from ._exceptions import LoadError, ImplError, RemoteError +from ._exceptions import ImplError, LoadError, RemoteError from .exceptions import LoadErrorReason from .types import FastEnum diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py index d689d6e25..a40adec22 100644 --- a/src/buildstream/_scheduler/__init__.py +++ b/src/buildstream/_scheduler/__init__.py @@ -17,14 +17,12 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +from .jobs import ElementJob, JobStatus from .queues import Queue, QueueStatus - +from .queues.artifactpushqueue import ArtifactPushQueue +from .queues.buildqueue import BuildQueue from .queues.fetchqueue import FetchQueue +from .queues.pullqueue import PullQueue from .queues.sourcepushqueue import SourcePushQueue from .queues.trackqueue import TrackQueue -from .queues.buildqueue import BuildQueue -from .queues.artifactpushqueue import ArtifactPushQueue -from .queues.pullqueue import PullQueue - -from .scheduler import Scheduler, SchedStatus, Notification, NotificationType -from .jobs import ElementJob, JobStatus +from .scheduler import Notification, NotificationType, SchedStatus, Scheduler diff --git a/src/buildstream/_scheduler/jobs/elementjob.py b/src/buildstream/_scheduler/jobs/elementjob.py index 6e035be9c..e102db18c 100644 --- a/src/buildstream/_scheduler/jobs/elementjob.py +++ b/src/buildstream/_scheduler/jobs/elementjob.py @@ -19,8 +19,7 @@ from ruamel import yaml from ..._message import MessageType - -from .job import Job, ChildJob +from .job import ChildJob, Job # ElementJob() diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py index 78a375fec..d9a39b06a 100644 --- a/src/buildstream/_scheduler/jobs/job.py +++ b/src/buildstream/_scheduler/jobs/job.py @@ -29,11 +29,11 @@ import signal import sys import traceback +from ... import _signals, utils # BuildStream toplevel imports -from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob +from ..._exceptions import BstError, ImplError, SkipJob, set_last_task_error from ..._message import Message, MessageType, unconditional_messages from ...types import FastEnum -from ... import _signals, utils from .. import _multiprocessing diff --git a/src/buildstream/_scheduler/queues/artifactpushqueue.py b/src/buildstream/_scheduler/queues/artifactpushqueue.py index 071c6fe74..1733960b3 100644 --- a/src/buildstream/_scheduler/queues/artifactpushqueue.py +++ b/src/buildstream/_scheduler/queues/artifactpushqueue.py @@ -18,10 +18,10 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> +from ..._exceptions import SkipJob +from ..resources import ResourceType # Local imports from . import Queue, QueueStatus -from ..resources import ResourceType -from ..._exceptions import SkipJob # A queue which pushes element artifacts diff --git a/src/buildstream/_scheduler/queues/buildqueue.py b/src/buildstream/_scheduler/queues/buildqueue.py index 5cbd5af57..95cce9d30 100644 --- a/src/buildstream/_scheduler/queues/buildqueue.py +++ b/src/buildstream/_scheduler/queues/buildqueue.py @@ -20,10 +20,10 @@ from datetime import timedelta -from . import Queue, QueueStatus -from ..resources import ResourceType from ..._message import MessageType from ..jobs import JobStatus +from ..resources import ResourceType +from . import Queue, QueueStatus # A queue which assembles elements diff --git a/src/buildstream/_scheduler/queues/fetchqueue.py b/src/buildstream/_scheduler/queues/fetchqueue.py index 18bf392d3..0af00dc4a 100644 --- a/src/buildstream/_scheduler/queues/fetchqueue.py +++ b/src/buildstream/_scheduler/queues/fetchqueue.py @@ -18,10 +18,10 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> +from ..jobs import JobStatus +from ..resources import ResourceType # Local imports from . import Queue, QueueStatus -from ..resources import ResourceType -from ..jobs import JobStatus # A queue which fetches element sources diff --git a/src/buildstream/_scheduler/queues/pullqueue.py b/src/buildstream/_scheduler/queues/pullqueue.py index e1d69590f..757619a76 100644 --- a/src/buildstream/_scheduler/queues/pullqueue.py +++ b/src/buildstream/_scheduler/queues/pullqueue.py @@ -18,11 +18,11 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> +from ..._exceptions import SkipJob +from ..jobs import JobStatus +from ..resources import ResourceType # Local imports from . import Queue, QueueStatus -from ..resources import ResourceType -from ..jobs import JobStatus -from ..._exceptions import SkipJob # A queue which pulls element artifacts diff --git a/src/buildstream/_scheduler/queues/queue.py b/src/buildstream/_scheduler/queues/queue.py index 986ac6c0a..f9848fc60 100644 --- a/src/buildstream/_scheduler/queues/queue.py +++ b/src/buildstream/_scheduler/queues/queue.py @@ -18,21 +18,20 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> +import heapq # System imports import os -from collections import deque -import heapq import traceback +from collections import deque from typing import TYPE_CHECKING -# Local imports -from ..jobs import ElementJob, JobStatus -from ..resources import ResourceType - # BuildStream toplevel imports from ..._exceptions import BstError, ImplError, set_last_task_error from ..._message import Message, MessageType from ...types import FastEnum +# Local imports +from ..jobs import ElementJob, JobStatus +from ..resources import ResourceType if TYPE_CHECKING: from typing import List, Optional diff --git a/src/buildstream/_scheduler/queues/sourcepushqueue.py b/src/buildstream/_scheduler/queues/sourcepushqueue.py index 897122108..c1cc26760 100644 --- a/src/buildstream/_scheduler/queues/sourcepushqueue.py +++ b/src/buildstream/_scheduler/queues/sourcepushqueue.py @@ -17,9 +17,9 @@ # Authors: # Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk> -from . import Queue, QueueStatus -from ..resources import ResourceType from ..._exceptions import SkipJob +from ..resources import ResourceType +from . import Queue, QueueStatus # A queue which pushes staged sources diff --git a/src/buildstream/_scheduler/queues/trackqueue.py b/src/buildstream/_scheduler/queues/trackqueue.py index d9c31ace1..6657ce669 100644 --- a/src/buildstream/_scheduler/queues/trackqueue.py +++ b/src/buildstream/_scheduler/queues/trackqueue.py @@ -20,11 +20,10 @@ # BuildStream toplevel imports from ...plugin import Plugin - +from ..jobs import JobStatus +from ..resources import ResourceType # Local imports from . import Queue, QueueStatus -from ..resources import ResourceType -from ..jobs import JobStatus # A queue which tracks sources diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py index 66174ad19..60a684327 100644 --- a/src/buildstream/_scheduler/scheduler.py +++ b/src/buildstream/_scheduler/scheduler.py @@ -19,22 +19,21 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> +import asyncio +import datetime # System imports import os -import asyncio -from itertools import chain import signal -import datetime import sys +from itertools import chain -# Local imports -from .resources import Resources -from .jobs import JobStatus -from ..types import FastEnum -from .._profile import Topics, PROFILER from .._message import Message, MessageType +from .._profile import PROFILER, Topics from ..plugin import Plugin - +from ..types import FastEnum +from .jobs import JobStatus +# Local imports +from .resources import Resources _MAX_TIMEOUT_TO_KILL_CHILDREN = 20 # in seconds diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py index 03b55b052..d760aaf71 100644 --- a/src/buildstream/_signals.py +++ b/src/buildstream/_signals.py @@ -21,11 +21,10 @@ import signal import sys import threading import traceback -from contextlib import contextmanager, ExitStack from collections import deque +from contextlib import ExitStack, contextmanager from typing import Callable, Deque - # Global per process state for handling of sigterm/sigtstp/sigcont, # note that it is expected that this only ever be used by new processes # the scheduler starts, not the main process. diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index dcde0b426..cd485cef1 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -18,15 +18,16 @@ # Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk> # import os + import grpc -from ._remote import BaseRemote -from ._cas.casremote import BlobNotFound -from .storage._casbaseddirectory import CasBasedDirectory -from ._basecache import BaseCache -from ._exceptions import CASError, CASRemoteError, SourceCacheError, RemoteError from . import utils +from ._basecache import BaseCache +from ._cas.casremote import BlobNotFound +from ._exceptions import CASError, CASRemoteError, RemoteError, SourceCacheError from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, source_pb2, source_pb2_grpc +from ._remote import BaseRemote +from .storage._casbaseddirectory import CasBasedDirectory class SourceRemote(BaseRemote): diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 3d646a756..ffe2b6360 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -20,41 +20,40 @@ # Tristan Maat <tristan.maat@codethink.co.uk> import os -import sys -import stat import shlex import shutil +import stat +import sys import tarfile import tempfile +from collections import deque from contextlib import contextmanager, suppress from fnmatch import fnmatch -from collections import deque from typing import List, Tuple -from ._artifactelement import verify_artifact_ref, ArtifactElement -from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError +from . import Scope, _site, _yaml, utils +from ._artifactelement import ArtifactElement, verify_artifact_ref +from ._exceptions import ArtifactElementError, ArtifactError, BstError, ImplError, StreamError from ._message import Message, MessageType +from ._pipeline import Pipeline +from ._profile import PROFILER, Topics from ._scheduler import ( - Scheduler, - SchedStatus, - TrackQueue, - FetchQueue, - SourcePushQueue, - BuildQueue, - PullQueue, ArtifactPushQueue, - NotificationType, - Notification, + BuildQueue, + FetchQueue, JobStatus, + Notification, + NotificationType, + PullQueue, + SchedStatus, + Scheduler, + SourcePushQueue, + TrackQueue, ) -from .element import Element -from ._pipeline import Pipeline -from ._profile import Topics, PROFILER from ._state import State -from .types import _KeyStrength, _PipelineSelection, _SchedulerErrorAction +from .element import Element from .plugin import Plugin -from . import utils, _yaml, _site -from . import Scope +from .types import _KeyStrength, _PipelineSelection, _SchedulerErrorAction # Stream() diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py index a54a17ff1..f79f2eedc 100644 --- a/src/buildstream/_workspaces.py +++ b/src/buildstream/_workspaces.py @@ -18,13 +18,11 @@ # Tristan Maat <tristan.maat@codethink.co.uk> import os -from . import utils -from . import _yaml +from . import _yaml, utils from ._exceptions import LoadError from .exceptions import LoadErrorReason - BST_WORKSPACE_FORMAT_VERSION = 4 BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1 WORKSPACE_PROJECT_FILE = ".bstproject.yaml" diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py index aa427064b..4c30b0233 100644 --- a/src/buildstream/buildelement.py +++ b/src/buildstream/buildelement.py @@ -141,7 +141,6 @@ from .element import Element from .sandbox import SandboxFlags from .types import Scope - # This list is preserved because of an unfortunate situation, we # need to remove these older commands which were secret and never # documented, but without breaking the cache keys. diff --git a/src/buildstream/downloadablefilesource.py b/src/buildstream/downloadablefilesource.py index 7c2da1c02..40164caf2 100644 --- a/src/buildstream/downloadablefilesource.py +++ b/src/buildstream/downloadablefilesource.py @@ -32,15 +32,15 @@ implementation. """ -import os -import urllib.request -import urllib.error import contextlib -import shutil import netrc +import os +import shutil +import urllib.error +import urllib.request -from .source import Source, SourceError from . import utils +from .source import Source, SourceError class _NetrcFTPOpener(urllib.request.FTPHandler): diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 6a0fa5fab..2fb3965c8 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -72,42 +72,37 @@ Class Reference --------------- """ +import contextlib +import copy import os import re import stat -import copy +import string import warnings from collections import OrderedDict -import contextlib from contextlib import contextmanager from functools import partial from itertools import chain -import string -from typing import cast, TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, cast from pyroaring import BitMap # pylint: disable=no-name-in-module -from . import _yaml +from . import _cachekey, _site, _yaml, utils +from ._artifact import Artifact +from ._exceptions import BstError, ImplError, LoadError, SourceCacheError from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION -from ._exceptions import BstError, LoadError, ImplError, SourceCacheError from .exceptions import ErrorDomain, LoadErrorReason -from .utils import FileListResult, BST_ARBITRARY_TIMESTAMP -from . import utils -from . import _cachekey -from . import _site from .node import Node from .plugin import Plugin -from .sandbox import SandboxFlags, SandboxCommandError +from .sandbox import SandboxCommandError, SandboxFlags from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote -from .types import CoreWarnings, Scope, _CacheBuildTrees, _KeyStrength -from ._artifact import Artifact - -from .storage.directory import Directory -from .storage._filebaseddirectory import FileBasedDirectory from .storage._casbaseddirectory import CasBasedDirectory -from .storage.directory import VirtualDirectoryError +from .storage._filebaseddirectory import FileBasedDirectory +from .storage.directory import Directory, VirtualDirectoryError +from .types import CoreWarnings, Scope, _CacheBuildTrees, _KeyStrength +from .utils import BST_ARBITRARY_TIMESTAMP, FileListResult if TYPE_CHECKING: from .node import MappingNode, ScalarNode, SequenceNode diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index 14e22e56d..e2b13315c 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -114,11 +114,11 @@ import os import subprocess import sys from contextlib import contextmanager -from typing import Generator, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Generator, Optional, Tuple from weakref import WeakValueDictionary from . import utils -from ._exceptions import PluginError, ImplError +from ._exceptions import ImplError, PluginError from ._message import Message, MessageType from .node import MappingNode, ProvenanceInformation from .types import CoreWarnings, SourceRef diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py index 9c136df9d..86d33932d 100644 --- a/src/buildstream/plugins/elements/compose.py +++ b/src/buildstream/plugins/elements/compose.py @@ -34,6 +34,7 @@ The default configuration and possible options are as such: """ import os + from buildstream import Element, Scope diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py index de7ee8af4..133edad0d 100644 --- a/src/buildstream/plugins/elements/import.py +++ b/src/buildstream/plugins/elements/import.py @@ -31,6 +31,7 @@ The empty configuration is as such: """ import os + from buildstream import Element, ElementError diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py index 93b554472..e8e000df9 100644 --- a/src/buildstream/plugins/sources/bzr.py +++ b/src/buildstream/plugins/sources/bzr.py @@ -54,13 +54,12 @@ See :ref:`built-in functionality doumentation <core_source_builtins>` for details on common configuration options for sources. """ +import fcntl import os import shutil -import fcntl from contextlib import contextmanager -from buildstream import Source, SourceError -from buildstream import utils +from buildstream import Source, SourceError, utils class BzrSource(Source): diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py index ffcae4993..7b4f5595c 100644 --- a/src/buildstream/plugins/sources/local.py +++ b/src/buildstream/plugins/sources/local.py @@ -37,8 +37,9 @@ details on common configuration options for sources. """ import os -from buildstream.storage.directory import Directory + from buildstream import Source, SourceError +from buildstream.storage.directory import Directory class LocalSource(Source): diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py index 18672df8a..62a79fb29 100644 --- a/src/buildstream/plugins/sources/patch.py +++ b/src/buildstream/plugins/sources/patch.py @@ -45,8 +45,8 @@ details on common configuration options for sources. """ import os -from buildstream import Source, SourceError -from buildstream import utils + +from buildstream import Source, SourceError, utils class PatchSource(Source): diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py index 9ed3099d9..ab89bb152 100644 --- a/src/buildstream/plugins/sources/remote.py +++ b/src/buildstream/plugins/sources/remote.py @@ -48,6 +48,7 @@ See :ref:`built-in functionality doumentation <core_source_builtins>` for details on common configuration options for sources. """ import os + from buildstream import DownloadableFileSource, SourceError, utils diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index aba927b99..3f455b40b 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -60,8 +60,7 @@ import tarfile from contextlib import contextmanager from tempfile import TemporaryFile -from buildstream import DownloadableFileSource, SourceError -from buildstream import utils +from buildstream import DownloadableFileSource, SourceError, utils class ReadableTarInfo(tarfile.TarInfo): diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py index 44d0889b3..129df21b9 100644 --- a/src/buildstream/plugins/sources/workspace.py +++ b/src/buildstream/plugins/sources/workspace.py @@ -37,10 +37,10 @@ workspace. The node constructed would be specified as follows: import os -from buildstream.storage.directory import Directory from buildstream import Source, SourceError -from buildstream.types import SourceRef from buildstream.node import MappingNode +from buildstream.storage.directory import Directory +from buildstream.types import SourceRef class WorkspaceSource(Source): diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py index 116425934..3e5a273cf 100644 --- a/src/buildstream/plugins/sources/zip.py +++ b/src/buildstream/plugins/sources/zip.py @@ -57,11 +57,10 @@ details on common configuration options for sources. """ import os -import zipfile import stat +import zipfile -from buildstream import DownloadableFileSource, SourceError -from buildstream import utils +from buildstream import DownloadableFileSource, SourceError, utils class ZipSource(DownloadableFileSource): diff --git a/src/buildstream/sandbox/__init__.py b/src/buildstream/sandbox/__init__.py index 5966d194f..35846fee0 100644 --- a/src/buildstream/sandbox/__init__.py +++ b/src/buildstream/sandbox/__init__.py @@ -17,6 +17,6 @@ # Authors: # Tristan Maat <tristan.maat@codethink.co.uk> -from .sandbox import Sandbox, SandboxFlags, SandboxCommandError -from ._sandboxremote import SandboxRemote from ._sandboxdummy import SandboxDummy +from ._sandboxremote import SandboxRemote +from .sandbox import Sandbox, SandboxCommandError, SandboxFlags diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py index 3d71b7440..607c2d1ce 100644 --- a/src/buildstream/sandbox/_sandboxbuildboxrun.py +++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py @@ -22,12 +22,12 @@ from contextlib import ExitStack import psutil -from .. import utils, _signals -from . import SandboxFlags +from .. import _signals, utils from .._exceptions import SandboxError from .._message import Message, MessageType from .._platform import Platform from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 +from . import SandboxFlags from ._sandboxreapi import SandboxREAPI diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index 5c2851580..2e3e29a5c 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -17,10 +17,10 @@ import os import shlex -from .sandbox import Sandbox, SandboxFlags, SandboxCommandError, _SandboxBatch from .. import utils from .._exceptions import ImplError, SandboxError from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 +from .sandbox import Sandbox, SandboxCommandError, SandboxFlags, _SandboxBatch # SandboxREAPI() diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 41fb9e153..abcb482d9 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -21,23 +21,21 @@ import os import shutil from collections import namedtuple -from urllib.parse import urlparse from functools import partial +from urllib.parse import urlparse import grpc -from .. import utils -from ..node import Node +from .. import _signals, _yaml, utils +from .._cas import CASRemote +from .._exceptions import BstError, SandboxError from .._message import Message, MessageType -from ._sandboxreapi import SandboxREAPI -from .. import _signals from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc -from .._protos.google.rpc import code_pb2 -from .._exceptions import BstError, SandboxError -from .. import _yaml from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc -from .._cas import CASRemote +from .._protos.google.rpc import code_pb2 from .._remote import RemoteSpec +from ..node import Node +from ._sandboxreapi import SandboxREAPI class RemoteExecutionSpec(namedtuple("RemoteExecutionSpec", "exec_service storage_service action_service")): diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py index 592866dac..3465e5eda 100644 --- a/src/buildstream/sandbox/sandbox.py +++ b/src/buildstream/sandbox/sandbox.py @@ -29,16 +29,16 @@ conform to this interface. See also: :ref:`sandboxing`. """ +import contextlib import os import shlex -import contextlib from contextlib import contextmanager -from typing import Dict, Generator, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Generator, List, Optional -from .._exceptions import ImplError, BstError, SandboxError +from .._exceptions import BstError, ImplError, SandboxError from .._message import Message, MessageType -from ..storage.directory import Directory from ..storage._casbaseddirectory import CasBasedDirectory +from ..storage.directory import Directory if TYPE_CHECKING: from typing import Union diff --git a/src/buildstream/scriptelement.py b/src/buildstream/scriptelement.py index f8deff28e..270328cba 100644 --- a/src/buildstream/scriptelement.py +++ b/src/buildstream/scriptelement.py @@ -34,7 +34,7 @@ implementations. import os from collections import OrderedDict -from typing import List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional from .element import Element, ElementError from .sandbox import SandboxFlags diff --git a/src/buildstream/source.py b/src/buildstream/source.py index f15d5a628..9726d257b 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -159,21 +159,20 @@ Class Reference import os from contextlib import contextmanager -from typing import Iterable, Iterator, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Iterable, Iterator, Optional, Tuple from . import _yaml, utils -from .node import MappingNode -from .plugin import Plugin -from .types import SourceRef, Union, List +from ._cachekey import generate_key from ._exceptions import BstError, ImplError, PluginError -from .exceptions import ErrorDomain from ._loader.metasource import MetaSource from ._projectrefs import ProjectRefStorage -from ._cachekey import generate_key -from .storage import CasBasedDirectory -from .storage import FileBasedDirectory -from .storage.directory import Directory, VirtualDirectoryError from ._variables import Variables +from .exceptions import ErrorDomain +from .node import MappingNode +from .plugin import Plugin +from .storage import CasBasedDirectory, FileBasedDirectory +from .storage.directory import Directory, VirtualDirectoryError +from .types import List, SourceRef, Union if TYPE_CHECKING: from typing import Any, Dict, Set diff --git a/src/buildstream/storage/__init__.py b/src/buildstream/storage/__init__.py index 5571cd807..d1fabe44e 100644 --- a/src/buildstream/storage/__init__.py +++ b/src/buildstream/storage/__init__.py @@ -18,6 +18,6 @@ # Authors: # Jim MacArthur <jim.macarthur@codethink.co.uk> -from ._filebaseddirectory import FileBasedDirectory from ._casbaseddirectory import CasBasedDirectory +from ._filebaseddirectory import FileBasedDirectory from .directory import Directory diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 72a6beeef..6a55ad64d 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -32,13 +32,14 @@ import stat import tarfile as tarfilelib from contextlib import contextmanager from io import StringIO + from google.protobuf import timestamp_pb2 from .. import utils from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 -from .directory import Directory, VirtualDirectoryError, _FileType +from ..utils import BST_ARBITRARY_TIMESTAMP, FileListResult from ._filebaseddirectory import FileBasedDirectory -from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP +from .directory import Directory, VirtualDirectoryError, _FileType class IndexEntry: diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py index 3f1fc599a..ad3befa5d 100644 --- a/src/buildstream/storage/_filebaseddirectory.py +++ b/src/buildstream/storage/_filebaseddirectory.py @@ -31,11 +31,18 @@ import os import shutil import stat -from .directory import Directory, VirtualDirectoryError, _FileType from .. import utils -from ..utils import link_files, copy_files, list_relative_paths, _get_link_mtime, BST_ARBITRARY_TIMESTAMP -from ..utils import _set_deterministic_user, _set_deterministic_mtime -from ..utils import FileListResult +from ..utils import ( + BST_ARBITRARY_TIMESTAMP, + FileListResult, + _get_link_mtime, + _set_deterministic_mtime, + _set_deterministic_user, + copy_files, + link_files, + list_relative_paths, +) +from .directory import Directory, VirtualDirectoryError, _FileType # FileBasedDirectory intentionally doesn't call its superclass constuctor, # which is meant to be unimplemented. diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py index f9ea4044a..5aed774ff 100644 --- a/src/buildstream/storage/directory.py +++ b/src/buildstream/storage/directory.py @@ -34,7 +34,7 @@ See also: :ref:`sandboxing`. import os import stat -from typing import Callable, Optional, Union, List +from typing import Callable, List, Optional, Union from .._exceptions import BstError from ..exceptions import ErrorDomain diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py index f09c5bda1..65b7b073d 100644 --- a/src/buildstream/testing/__init__.py +++ b/src/buildstream/testing/__init__.py @@ -21,13 +21,15 @@ This package contains various utilities which make it easier to test plugins. import os from collections import OrderedDict -from buildstream.exceptions import ErrorDomain, LoadErrorReason + from buildstream._yaml import load as load_yaml # type: ignore -from ._yaml import generate_project, generate_element +from buildstream.exceptions import ErrorDomain, LoadErrorReason + +from ._cachekeys import check_cache_key_stability +from ._yaml import generate_element, generate_project +from .integration import integration_cache from .repo import Repo from .runcli import cli, cli_integration, cli_remote_execution -from .integration import integration_cache -from ._cachekeys import check_cache_key_stability __all__ = [ "check_cache_key_stability", diff --git a/src/buildstream/testing/_fixtures.py b/src/buildstream/testing/_fixtures.py index 5da51bb45..dd7c5e9d2 100644 --- a/src/buildstream/testing/_fixtures.py +++ b/src/buildstream/testing/_fixtures.py @@ -19,6 +19,7 @@ import pytest from buildstream import node, utils + # Catch tests that don't shut down background threads, which could then lead # to other tests hanging when BuildStream uses fork(). @pytest.fixture(autouse=True) diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py index 782d99814..76b97ab30 100644 --- a/src/buildstream/testing/_sourcetests/build_checkout.py +++ b/src/buildstream/testing/_sourcetests/build_checkout.py @@ -20,11 +20,13 @@ # pylint: disable=redefined-outer-name import os + import pytest -from buildstream.testing import create_repo -from buildstream.testing import cli # pylint: disable=unused-import from buildstream import _yaml +from buildstream.testing import cli # pylint: disable=unused-import +from buildstream.testing import create_repo + from .utils import kind # pylint: disable=unused-import # Project directory diff --git a/src/buildstream/testing/_sourcetests/conftest.py b/src/buildstream/testing/_sourcetests/conftest.py index 64dd404ef..88b623dd4 100644 --- a/src/buildstream/testing/_sourcetests/conftest.py +++ b/src/buildstream/testing/_sourcetests/conftest.py @@ -14,4 +14,5 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. -from .._fixtures import reset_global_node_state, thread_check # pylint: disable=unused-import +from .._fixtures import reset_global_node_state # pylint: disable=unused-import +from .._fixtures import thread_check diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py index 05b43d793..fbc0df0ed 100644 --- a/src/buildstream/testing/_sourcetests/fetch.py +++ b/src/buildstream/testing/_sourcetests/fetch.py @@ -20,15 +20,16 @@ # pylint: disable=redefined-outer-name import os + import pytest from buildstream import _yaml -from .._utils import generate_junction -from .. import create_repo + from .. import cli # pylint: disable=unused-import -from .utils import update_project_configuration +from .. import create_repo +from .._utils import generate_junction from .utils import kind # pylint: disable=unused-import - +from .utils import update_project_configuration # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py index 69042747c..fd076121e 100644 --- a/src/buildstream/testing/_sourcetests/mirror.py +++ b/src/buildstream/testing/_sourcetests/mirror.py @@ -20,13 +20,15 @@ # pylint: disable=redefined-outer-name import os + import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain -from .._utils import generate_junction -from .. import create_repo + from .. import cli # pylint: disable=unused-import +from .. import create_repo +from .._utils import generate_junction from .utils import kind # pylint: disable=unused-import # Project directory diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py index b834f3223..e7f308031 100644 --- a/src/buildstream/testing/_sourcetests/source_determinism.py +++ b/src/buildstream/testing/_sourcetests/source_determinism.py @@ -20,12 +20,14 @@ # pylint: disable=redefined-outer-name import os + import pytest from buildstream import _yaml -from .._utils.site import HAVE_SANDBOX, CASD_SEPARATE_USER -from .. import create_repo + from .. import cli # pylint: disable=unused-import +from .. import create_repo +from .._utils.site import CASD_SEPARATE_USER, HAVE_SANDBOX from .utils import kind # pylint: disable=unused-import # Project directory diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py index ecb508b1a..25a2b911a 100644 --- a/src/buildstream/testing/_sourcetests/track.py +++ b/src/buildstream/testing/_sourcetests/track.py @@ -20,16 +20,17 @@ # pylint: disable=redefined-outer-name import os + import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain -from .._utils import generate_junction -from .. import create_repo + from .. import cli # pylint: disable=unused-import -from .utils import update_project_configuration +from .. import create_repo +from .._utils import generate_junction from .utils import kind # pylint: disable=unused-import - +from .utils import update_project_configuration # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py index 2c4141936..89917ac45 100644 --- a/src/buildstream/testing/_sourcetests/track_cross_junction.py +++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py @@ -24,12 +24,12 @@ import os import pytest from buildstream import _yaml -from .._utils import generate_junction -from .. import create_repo, ALL_REPO_KINDS + from .. import cli # pylint: disable=unused-import +from .. import ALL_REPO_KINDS, create_repo +from .._utils import generate_junction from .utils import add_plugins_conf - # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") diff --git a/src/buildstream/testing/_sourcetests/utils.py b/src/buildstream/testing/_sourcetests/utils.py index 4cfb696bd..1331ae101 100644 --- a/src/buildstream/testing/_sourcetests/utils.py +++ b/src/buildstream/testing/_sourcetests/utils.py @@ -21,6 +21,10 @@ import os +from buildstream import _yaml + +from .. import ALL_REPO_KINDS + # To make use of these test utilities it is necessary to have pytest # available. However, we don't want to have a hard dependency on # pytest. @@ -31,9 +35,6 @@ except ImportError: msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name) raise ImportError(msg) -from buildstream import _yaml -from .. import ALL_REPO_KINDS - # kind() # diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py index 3520a8cc4..35418afcb 100644 --- a/src/buildstream/testing/_sourcetests/workspace.py +++ b/src/buildstream/testing/_sourcetests/workspace.py @@ -21,11 +21,13 @@ import os import shutil + import pytest from buildstream import _yaml -from .. import create_repo + from .. import cli # pylint: disable=unused-import +from .. import create_repo from .utils import kind # pylint: disable=unused-import # Project directory diff --git a/src/buildstream/testing/_update_cachekeys.py b/src/buildstream/testing/_update_cachekeys.py index 219e17f8b..e588f98ea 100755 --- a/src/buildstream/testing/_update_cachekeys.py +++ b/src/buildstream/testing/_update_cachekeys.py @@ -32,7 +32,7 @@ import os import tempfile from unittest import mock -from buildstream.testing._cachekeys import _element_filename, _parse_output_keys, _load_expected_keys +from buildstream.testing._cachekeys import _element_filename, _load_expected_keys, _parse_output_keys from buildstream.testing.runcli import Cli diff --git a/src/buildstream/testing/_utils/junction.py b/src/buildstream/testing/_utils/junction.py index 8f80ed6bd..8e48254fd 100644 --- a/src/buildstream/testing/_utils/junction.py +++ b/src/buildstream/testing/_utils/junction.py @@ -1,9 +1,11 @@ import subprocess + import pytest from buildstream import _yaml + from .. import Repo -from .site import HAVE_GIT, GIT, GIT_ENV +from .site import GIT, GIT_ENV, HAVE_GIT # generate_junction() diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index d8ace859b..a9b2f532c 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -7,10 +7,9 @@ import subprocess import sys from typing import Optional # pylint: disable=unused-import -from buildstream import utils, ProgramNotFoundError +from buildstream import ProgramNotFoundError, utils from buildstream._platform import Platform - try: GIT = utils.get_host_tool("git") # type: Optional[str] HAVE_GIT = True diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py index 1e868609a..9cb2667e5 100644 --- a/src/buildstream/testing/runcli.py +++ b/src/buildstream/testing/runcli.py @@ -29,17 +29,16 @@ runcli - Test fixtures used for running BuildStream commands """ +import itertools import os import re -import sys import shutil +import sys import tempfile -import itertools import traceback -from contextlib import contextmanager, ExitStack -from ruamel import yaml -import pytest +from contextlib import ExitStack, contextmanager +import pytest # XXX Using pytest private internals here # # We use pytest internals to capture the stdout/stderr during @@ -47,17 +46,17 @@ import pytest # CliRunner convenience API (click.testing module) does not support # separation of stdout/stderr. # -from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary +from _pytest.capture import FDCapture, FDCaptureBinary, MultiCapture +from ruamel import yaml -# Import the main cli entrypoint -from buildstream._frontend import cli as bst_cli from buildstream import _yaml, node from buildstream._cas import CASCache -from buildstream.element import _get_normal_name, _compose_artifact_name - # Special private exception accessor, for test case purposes from buildstream._exceptions import BstError, get_last_exception, get_last_task_error +# Import the main cli entrypoint +from buildstream._frontend import cli as bst_cli from buildstream._protos.buildstream.v2 import artifact_pb2 +from buildstream.element import _compose_artifact_name, _get_normal_name # Wrapper for the click.testing result diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py index 9c6761ccc..0d3d842ab 100644 --- a/src/buildstream/utils.py +++ b/src/buildstream/utils.py @@ -22,36 +22,35 @@ Utilities """ import calendar +import datetime import errno import hashlib +import itertools import math import os import re import shutil import signal import stat -from stat import S_ISDIR import subprocess import tempfile import time -import datetime -import itertools from contextlib import contextmanager from pathlib import Path -from typing import Callable, IO, Iterable, Iterator, Optional, Tuple, Union -from dateutil import parser as dateutil_parser -from google.protobuf import timestamp_pb2 +from stat import S_ISDIR +from typing import IO, Callable, Iterable, Iterator, Optional, Tuple, Union import psutil +from dateutil import parser as dateutil_parser +from google.protobuf import timestamp_pb2 from . import _signals from ._exceptions import BstError -from .exceptions import ErrorDomain from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 - # Contains utils that have been rewritten in Cython for speed benefits # This makes them available when importing from utils from ._utils import url_directory_name # pylint: disable=unused-import +from .exceptions import ErrorDomain # The magic number for timestamps: 2011-11-11 11:11:11 BST_ARBITRARY_TIMESTAMP = calendar.timegm((2011, 11, 11, 11, 11, 11)) |