diff options
author | Daniel Firth <dan.firth@codethink.co.uk> | 2016-11-29 13:01:20 +0000 |
---|---|---|
committer | Daniel Firth <dan.firth@codethink.co.uk> | 2016-11-29 13:01:20 +0000 |
commit | a3e53b4232be68b3748f3985d2f5191cb0f7241d (patch) | |
tree | 86b8b498ea9b836865d12cf5ab5fa758beb99462 | |
parent | 54aff1da1ea84c78e768681157f053c753c4df36 (diff) | |
download | ybd-a3e53b4232be68b3748f3985d2f5191cb0f7241d.tar.gz |
Revert "WIP: Put the config in a different file?"
This reverts commit ad53dd542e43b8f4c6c55fad8452a0962f842834.
-rwxr-xr-x | kbas/__main__.py | 52 | ||||
-rw-r--r-- | ybd/__init__.py | 1 | ||||
-rwxr-xr-x | ybd/__main__.py | 3 | ||||
-rw-r--r-- | ybd/app.py | 2 | ||||
-rw-r--r-- | ybd/cache.py | 90 | ||||
-rw-r--r-- | ybd/config.py | 1 | ||||
-rw-r--r-- | ybd/defaults.py | 3 | ||||
-rw-r--r-- | ybd/deployment.py | 2 | ||||
-rw-r--r-- | ybd/morphs.py | 2 | ||||
-rw-r--r-- | ybd/release_note.py | 3 | ||||
-rw-r--r-- | ybd/repos.py | 18 | ||||
-rw-r--r-- | ybd/sandbox.py | 44 | ||||
-rw-r--r-- | ybd/utils.py | 5 |
13 files changed, 110 insertions, 116 deletions
diff --git a/kbas/__main__.py b/kbas/__main__.py index 204c7b9..4445f67 100755 --- a/kbas/__main__.py +++ b/kbas/__main__.py @@ -40,9 +40,9 @@ class KeyedBinaryArtifactServer(object): app.load_configs([ os.path.join(os.getcwd(), 'kbas.conf'), os.path.join(os.path.dirname(__file__), 'config', 'kbas.conf')]) - config.config['start-time'] = datetime.now() - config.config['last-upload'] = datetime.now() - config.config['downloads'] = 0 + app.config['start-time'] = datetime.now() + app.config['last-upload'] = datetime.now() + app.config['downloads'] = 0 try: import cherrypy @@ -51,12 +51,12 @@ class KeyedBinaryArtifactServer(object): server = 'wsgiref' # for development: - if config.config.get('mode') == 'development': - bottle.run(server=server, host=config.config['host'], - port=config.config['port'], debug=True, reloader=True) + if app.config.get('mode') == 'development': + bottle.run(server=server, host=app.config['host'], + port=app.config['port'], debug=True, reloader=True) else: - bottle.run(server=server, host=config.config['host'], - port=config.config['port'], reloader=True) + bottle.run(server=server, host=app.config['host'], + port=app.config['port'], reloader=True) @bottle.get('/static/<filename>') def send_static(filename): @@ -67,7 +67,7 @@ class KeyedBinaryArtifactServer(object): @bottle.get('/<name>') @bottle.get('/artifacts/<name>') def list(name=""): - names = glob.glob(os.path.join(config.config['artifact-dir'], + names = glob.glob(os.path.join(app.config['artifact-dir'], '*' + name + '*')) try: content = [[strftime('%y-%m-%d', gmtime(os.stat(x).st_atime)), @@ -86,28 +86,28 @@ class KeyedBinaryArtifactServer(object): @bottle.get('/1.0/artifacts') def get_morph_artifact(): f = request.query.filename - return static_file(f, root=config.config['artifact-dir'], download=True) + return static_file(f, root=app.config['artifact-dir'], download=True) @bottle.get('/get/<cache_id>') def get_artifact(cache_id): f = os.path.join(cache_id, cache_id) - config.config['downloads'] += 1 - return static_file(f, root=config.config['artifact-dir'], download=True, + app.config['downloads'] += 1 + return static_file(f, root=app.config['artifact-dir'], download=True, mimetype='application/x-tar') @bottle.get('/') @bottle.get('/status') def status(): - stat = os.statvfs(config.config['artifact-dir']) + stat = os.statvfs(app.config['artifact-dir']) free = stat.f_frsize * stat.f_bavail / 1000000000 - artifacts = len(os.listdir(config.config['artifact-dir'])) - started = config.config['start-time'].strftime('%y-%m-%d %H:%M:%S') - downloads = config.config['downloads'] - last_upload = config.config['last-upload'].strftime('%y-%m-%d %H:%M:%S') + artifacts = len(os.listdir(app.config['artifact-dir'])) + started = app.config['start-time'].strftime('%y-%m-%d %H:%M:%S') + downloads = app.config['downloads'] + last_upload = app.config['last-upload'].strftime('%y-%m-%d %H:%M:%S') content = [['Started:', started, None]] content += [['Last upload:', last_upload, None]] - if config.config.get('last-reject'): - content += [['Last reject:', config.config['last-reject'], None]] + if app.config.get('last-reject'): + content += [['Last reject:', app.config['last-reject'], None]] content += [['Space:', str(free) + 'GB', None]] content += [['Artifacts:', str(artifacts), None]] content += [['Downloads:', downloads, None]] @@ -118,10 +118,10 @@ class KeyedBinaryArtifactServer(object): @bottle.post('/upload') def post_artifact(): - if config.config['password'] is 'insecure' or \ - request.forms.get('password') != config.config['password']: + if app.config['password'] is 'insecure' or \ + request.forms.get('password') != app.config['password']: print 'Upload attempt: password fail' - config.config['last-reject'] = \ + app.config['last-reject'] = \ datetime.now().strftime('%y-%m-%d %H:%M:%S') response.status = 401 # unauthorized return @@ -131,14 +131,14 @@ class KeyedBinaryArtifactServer(object): response.status = 400 # bad request, cache_id contains bad things return - if os.path.isdir(os.path.join(config.config['artifact-dir'], cache_id)): + if os.path.isdir(os.path.join(app.config['artifact-dir'], cache_id)): if cache.check(cache_id) == request.forms.get('checksum', 'XYZ'): response.status = 777 # this is the same binary we have return response.status = 405 # not allowed, this artifact exists return - tempfile.tempdir = config.config['artifact-dir'] + tempfile.tempdir = app.config['artifact-dir'] tmpdir = tempfile.mkdtemp() try: upload = request.files.get('file') @@ -158,10 +158,10 @@ class KeyedBinaryArtifactServer(object): checksum = cache.md5(artifact) with open(artifact + '.md5', "a") as f: f.write(checksum) - shutil.move(tmpdir, os.path.join(config.config['artifact-dir'], + shutil.move(tmpdir, os.path.join(app.config['artifact-dir'], cache_id)) response.status = 201 # success! - config.config['last-upload'] = datetime.now() + app.config['last-upload'] = datetime.now() return except: # something went wrong, clean up diff --git a/ybd/__init__.py b/ybd/__init__.py index 14839a6..ed7bb46 100644 --- a/ybd/__init__.py +++ b/ybd/__init__.py @@ -16,7 +16,6 @@ import ybd.app import ybd.assembly import ybd.cache -import ybd.config import ybd.defaults import ybd.morphs import ybd.pots diff --git a/ybd/__main__.py b/ybd/__main__.py index 7fb529e..59d5f83 100755 --- a/ybd/__main__.py +++ b/ybd/__main__.py @@ -21,9 +21,8 @@ import os import sys import fcntl from ybd import app, cache, sandbox -from ybd.app import cleanup, RetryException, setup, spawn, timer +from ybd.app import cleanup, config, log, RetryException, setup, spawn, timer from ybd.assembly import compose -from ybd.config import config from ybd.deployment import deploy from ybd.pots import Pots from ybd.concourse import Pipeline @@ -28,7 +28,6 @@ from fs.osfs import OSFS # not used here, but we import it to check install from ybd.repos import get_version from ybd.cache import cache_key from ybd.utils import log -from ybd.config import config try: from riemann_client.transport import TCPTransport from riemann_client.client import QueuedClient @@ -37,6 +36,7 @@ except ImportError: riemann_available = False +config = {} defs = {} diff --git a/ybd/cache.py b/ybd/cache.py index ae6b52a..7934384 100644 --- a/ybd/cache.py +++ b/ybd/cache.py @@ -43,17 +43,17 @@ def cache_key(dn): if dn.get('cache'): return dn['cache'] - if dn.get('arch', config.config['arch']) != config.config['arch']: + if dn.get('arch', app.config['arch']) != app.config['arch']: if 'tried' not in dn: dn['tried'] = True log(dn, 'No cache_key for arch %s mismatch' % dn['arch'], - config.config['arch']) + app.config['arch']) return False dn['cache'] = 'calculating' key = 'no-build' - if config.config.get('mode', 'normal') in ['keys-only', 'normal']: + if app.config.get('mode', 'normal') in ['keys-only', 'normal']: if dn.get('repo') and not dn.get('tree'): dn['tree'] = get_tree(dn) factors = hash_factors(dn) @@ -62,30 +62,30 @@ def cache_key(dn): dn['cache'] = dn['name'] + "." + key - config.config['total'] += 1 + app.config['total'] += 1 x = 'x' if not get_cache(dn): x = ' ' - config.config['tasks'] += 1 + app.config['tasks'] += 1 if dn.get('kind', 'chunk') == 'chunk': - config.config['chunks'] += 1 + app.config['chunks'] += 1 if dn.get('kind', 'chunk') == 'stratum': - config.config['strata'] += 1 + app.config['strata'] += 1 if dn.get('kind', 'chunk') == 'system': - config.config['systems'] += 1 + app.config['systems'] += 1 log('CACHE-KEYS', '[%s]' % x, dn['cache']) - if config.config.get('manifest', False): - update_manifest(dn, config.config['manifest']) + if app.config.get('manifest', False): + update_manifest(dn, app.config['manifest']) - if 'keys' in config.config: - config.config['keys'] += [dn['cache']] + if 'keys' in app.config: + app.config['keys'] += [dn['cache']] return dn['cache'] def hash_factors(dn): - hash_factors = {'arch': config.config['arch']} + hash_factors = {'arch': app.config['arch']} for factor in dn.get('build-depends', []): hash_factors[factor] = cache_key(factor) @@ -95,7 +95,7 @@ def hash_factors(dn): hash_factors[key] = cache_key(key) relevant_factors = ['tree', 'submodules'] + app.defs.defaults.build_steps - if config.config.get('artifact-version', False) not in range(0, 6): + if app.config.get('artifact-version', False) not in range(0, 6): relevant_factors += ['devices'] for factor in relevant_factors: @@ -103,8 +103,8 @@ def hash_factors(dn): hash_factors[factor] = dn[factor] if dn.get('kind') == 'system': - if config.config.get('default-splits', []) != []: - hash_factors['splits'] = config.config.get('default-splits') + if app.config.get('default-splits', []) != []: + hash_factors['splits'] = app.config.get('default-splits') def hash_system_recursively(system): factor = system.get('path', 'BROKEN') @@ -116,10 +116,10 @@ def hash_factors(dn): for system in dn.get('systems', []): hash_system_recursively(system) - if config.config.get('artifact-version', False): - hash_factors['artifact-version'] = config.config.get('artifact-version') + if app.config.get('artifact-version', False): + hash_factors['artifact-version'] = app.config.get('artifact-version') - if config.config.get('artifact-version', 0) in range(0, 2): + if app.config.get('artifact-version', 0) in range(0, 2): # this way, any change to any build-system invalidates all caches hash_factors['default-build-systems'] = \ app.defs.defaults.build_systems @@ -128,11 +128,11 @@ def hash_factors(dn): hash_factors['default-build-systems'] = \ app.defs.defaults.build_systems.get(dn.get('build-system', 'manual')) - if (config.config.get('default-splits', []) != [] and + if (app.config.get('default-splits', []) != [] and dn.get('kind') == 'system'): - hash_factors['default-splits'] = config.config['default-splits'] + hash_factors['default-splits'] = app.config['default-splits'] - if config.config.get('artifact-version', 0) not in range(0, 7): + if app.config.get('artifact-version', 0) not in range(0, 7): if dn.get('max-jobs'): if dn['max-jobs'] == 1: hash_factors['max-jobs'] = 'single' @@ -146,7 +146,7 @@ def cache(dn): if get_cache(dn): log(dn, "Bah! I could have cached", cache_key(dn)) return - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(dn)) if dn.get('kind') == "system": @@ -161,11 +161,11 @@ def cache(dn): shutil.move('%s.tar.gz' % cachefile, cachefile) unpack(dn, cachefile) - config.config['counter'].increment() + app.config['counter'].increment() - if config.config.get('kbas-password', 'insecure') != 'insecure' and \ - config.config.get('kbas-url') is not None: - if dn.get('kind', 'chunk') in config.config.get('kbas-upload', 'chunk'): + if app.config.get('kbas-password', 'insecure') != 'insecure' and \ + app.config.get('kbas-url') is not None: + if dn.get('kind', 'chunk') in app.config.get('kbas-upload', 'chunk'): with app.timer(dn, 'upload'): upload(dn) @@ -204,7 +204,7 @@ def unpack(dn, tmpfile): log(dn, 'Problem with tarfile', tmpfile, exit=True) try: - path = os.path.join(config.config['artifacts'], cache_key(dn)) + path = os.path.join(app.config['artifacts'], cache_key(dn)) shutil.move(os.path.dirname(tmpfile), path) if not os.path.isdir(path): log(dn, 'Problem creating artifact', path, exit=True) @@ -222,9 +222,9 @@ def unpack(dn, tmpfile): def upload(dn): cachefile = get_cache(dn) - url = config.config['kbas-url'] + 'upload' + url = app.config['kbas-url'] + 'upload' params = {"filename": dn['cache'], - "password": config.config['kbas-password'], + "password": app.config['kbas-password'], "checksum": md5(cachefile)} with open(cachefile, 'rb') as f: try: @@ -234,11 +234,11 @@ def upload(dn): return if response.status_code == 777: log(dn, 'Reproduced %s at' % md5(cachefile), dn['cache']) - config.config['reproduced'].append([md5(cachefile), dn['cache']]) + app.config['reproduced'].append([md5(cachefile), dn['cache']]) return if response.status_code == 405: # server has different md5 for this artifact - if dn['kind'] == 'stratum' and config.config['reproduce']: + if dn['kind'] == 'stratum' and app.config['reproduce']: log('BIT-FOR-BIT', 'WARNING: reproduction failed for', dn['cache']) log(dn, 'Artifact server already has', dn['cache']) @@ -255,13 +255,13 @@ def get_cache(dn): if cache_key(dn) is False: return False - cachedir = os.path.join(config.config['artifacts'], cache_key(dn)) + cachedir = os.path.join(app.config['artifacts'], cache_key(dn)) if os.path.isdir(cachedir): call(['touch', cachedir]) artifact = os.path.join(cachedir, cache_key(dn)) unpackdir = artifact + '.unpacked' if not os.path.isdir(unpackdir) and dn.get('kind') != 'system': - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() if call(['tar', 'xf', artifact, '--directory', tmpdir]): log(dn, 'Problem unpacking', artifact) @@ -280,26 +280,26 @@ def get_cache(dn): def get_remote(dn): ''' If a remote cached artifact exists for d, retrieve it ''' - if config.config.get('last-retry-component') == dn or dn.get('tried'): + if app.config.get('last-retry-component') == dn or dn.get('tried'): return False dn['tried'] = True # let's not keep asking for this artifact - if dn.get('kind', 'chunk') not in config.config.get('kbas-upload', 'chunk'): + if dn.get('kind', 'chunk') not in app.config.get('kbas-upload', 'chunk'): return False try: log(dn, 'Try downloading', cache_key(dn)) - url = config.config['kbas-url'] + 'get/' + cache_key(dn) + url = app.config['kbas-url'] + 'get/' + cache_key(dn) response = requests.get(url=url, stream=True) except: - config.config.pop('kbas-url') + app.config.pop('kbas-url') log(dn, 'WARNING: remote artifact server is not working') return False if response.status_code == 200: try: - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, cache_key(dn)) with open(cachefile, 'wb') as f: @@ -314,7 +314,7 @@ def get_remote(dn): def cull(artifact_dir): - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] deleted = 0 def clear(deleted, artifact_dir): @@ -322,7 +322,7 @@ def cull(artifact_dir): for artifact in artifacts: stat = os.statvfs(artifact_dir) free = stat.f_frsize * stat.f_bavail / 1000000000 - if free >= config.config.get('min-gigabytes', 10): + if free >= app.config.get('min-gigabytes', 10): log('SETUP', '%sGB is enough free space' % free) if deleted > 0: log('SETUP', 'Culled %s items in' % deleted, @@ -331,7 +331,7 @@ def cull(artifact_dir): path = os.path.join(artifact_dir, artifact) if os.path.exists(os.path.join(path, artifact + '.unpacked')): path = os.path.join(path, artifact + '.unpacked') - if os.path.exists(path) and artifact not in config.config['keys']: + if os.path.exists(path) and artifact not in app.config['keys']: tmpdir = tempfile.mkdtemp() shutil.move(path, os.path.join(tmpdir, 'to-delete')) app.remove_dir(tmpdir) @@ -348,14 +348,14 @@ def cull(artifact_dir): stat = os.statvfs(artifact_dir) free = stat.f_frsize * stat.f_bavail / 1000000000 - if free < config.config.get('min-gigabytes', 10): + if free < app.config.get('min-gigabytes', 10): log('SETUP', '%sGB is less than min-gigabytes:' % free, - config.config.get('min-gigabytes', 10), exit=True) + app.config.get('min-gigabytes', 10), exit=True) def check(artifact): try: - artifact = os.path.join(config.config['artifact-dir'], artifact, + artifact = os.path.join(app.config['artifact-dir'], artifact, artifact) checkfile = artifact + '.md5' if not os.path.exists(checkfile): diff --git a/ybd/config.py b/ybd/config.py deleted file mode 100644 index fcd8c8e..0000000 --- a/ybd/config.py +++ /dev/null @@ -1 +0,0 @@ -config = {} diff --git a/ybd/defaults.py b/ybd/defaults.py index 8d26f14..d263bdd 100644 --- a/ybd/defaults.py +++ b/ybd/defaults.py @@ -26,7 +26,6 @@ These definitions shall be used if no DEFAULTS file is present. import os from ybd import app -from ybd.config import config from ybd.utils import log import yaml @@ -44,7 +43,7 @@ class Defaults(object): def _load_defaults(self, defaults_file='./DEFAULTS'): '''Get defaults, either from a DEFAULTS file, or built-in defaults.''' ybd_defaults_file = os.path.join(os.path.dirname(__file__), - config.config['defaults']) + app.config['defaults']) ybd_defaults = self._load(ybd_defaults_file, ignore_errors=True) defaults = self._load(defaults_file, ignore_errors=True) diff --git a/ybd/deployment.py b/ybd/deployment.py index e8bee47..7d3b150 100644 --- a/ybd/deployment.py +++ b/ybd/deployment.py @@ -23,7 +23,7 @@ from ybd.utils import log def deploy(target): '''Deploy a cluster definition.''' - arch = config.config['arch'] + arch = app.config['arch'] for system in target.get('systems', []): if app.defs.get(system).get('arch', arch) == arch: with app.timer(system, 'deployment'): diff --git a/ybd/morphs.py b/ybd/morphs.py index edc29db..6c9df9f 100644 --- a/ybd/morphs.py +++ b/ybd/morphs.py @@ -17,7 +17,7 @@ import yaml import glob import os -from ybd.app import chdir +from ybd.app import chdir, donfig from ybd.config import config from ybd.defaults import Defaults from ybd.utils import log diff --git a/ybd/release_note.py b/ybd/release_note.py index fe6381f..4fbfa39 100644 --- a/ybd/release_note.py +++ b/ybd/release_note.py @@ -18,8 +18,7 @@ import os from subprocess import check_output import tempfile from ybd import app -from ybd.app import chdir -from ybd.config import config +from ybd.app import chdir, config from ybd.morphs import Morphs from ybd.repos import explore, get_last_tag, get_repo_name from ybd.repos import mirror, mirror_has_ref diff --git a/ybd/repos.py b/ybd/repos.py index ff2303e..c2c470d 100644 --- a/ybd/repos.py +++ b/ybd/repos.py @@ -38,7 +38,7 @@ else: def get_repo_url(repo): if repo: - for alias, url in config.config.get('aliases', {}).items(): + for alias, url in app.config.get('aliases', {}).items(): repo = repo.replace(alias, url) if repo[:4] == "http" and not repo.endswith('.git'): repo = repo + '.git' @@ -89,7 +89,7 @@ def get_last_tag(gitdir): def get_tree(dn): ref = str(dn['ref']) - gitdir = os.path.join(config.config['gits'], get_repo_name(dn['repo'])) + gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo'])) if dn['repo'].startswith('file://') or dn['repo'].startswith('/'): gitdir = dn['repo'].replace('file://', '') if not os.path.isdir(gitdir): @@ -98,10 +98,10 @@ def get_tree(dn): if not os.path.exists(gitdir): try: params = {'repo': get_repo_url(dn['repo']), 'ref': ref} - r = requests.get(url=config.config['tree-server'], params=params) + r = requests.get(url=app.config['tree-server'], params=params) return r.json()['tree'] except: - if config.config.get('tree-server'): + if app.config.get('tree-server'): log(dn, 'WARNING: no tree from tree-server for', ref) mirror(dn['name'], dn['repo']) @@ -125,7 +125,7 @@ def get_tree(dn): def mirror(name, repo): - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() repo_url = get_repo_url(repo) try: @@ -133,7 +133,7 @@ def mirror(name, repo): log(name, 'Try fetching tarball %s' % tar_file) # try tarball first with app.chdir(tmpdir), open(os.devnull, "w") as fnull: - call(['wget', os.path.join(config.config['tar-url'], tar_file)], + call(['wget', os.path.join(app.config['tar-url'], tar_file)], stdout=fnull, stderr=fnull) call(['tar', 'xf', tar_file], stderr=fnull) call(['git', 'config', 'gc.autodetach', 'false'], stderr=fnull) @@ -149,7 +149,7 @@ def mirror(name, repo): if call(['git', 'rev-parse']): log(name, 'Problem mirroring git repo at', tmpdir, exit=True) - gitdir = os.path.join(config.config['gits'], get_repo_name(repo)) + gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) try: shutil.move(tmpdir, gitdir) log(name, 'Git repo is mirrored at', gitdir) @@ -188,7 +188,7 @@ def checkout(dn): def _checkout(name, repo, ref, checkout): - gitdir = os.path.join(config.config['gits'], get_repo_name(repo)) + gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) if not os.path.exists(gitdir): mirror(name, repo) elif not mirror_has_ref(gitdir, ref): @@ -224,7 +224,7 @@ def extract_commit(name, repo, ref, target_dir): function is much quicker when you don't need to copy the whole repo into target_dir. ''' - gitdir = os.path.join(config.config['gits'], get_repo_name(repo)) + gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) if not os.path.exists(gitdir): mirror(name, repo) elif not mirror_has_ref(gitdir, ref): diff --git a/ybd/sandbox.py b/ybd/sandbox.py index 4e40d44..1f489c0 100644 --- a/ybd/sandbox.py +++ b/ybd/sandbox.py @@ -35,20 +35,20 @@ executor = None @contextlib.contextmanager def setup(dn): - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] dn['sandbox'] = tempfile.mkdtemp() - os.environ['TMPDIR'] = config.config['tmp'] - config.config['sandboxes'] += [dn['sandbox']] + os.environ['TMPDIR'] = app.config['tmp'] + app.config['sandboxes'] += [dn['sandbox']] dn['checkout'] = os.path.join(dn['sandbox'], dn['name'] + '.build') dn['install'] = os.path.join(dn['sandbox'], dn['name'] + '.inst') dn['baserockdir'] = os.path.join(dn['install'], 'baserock') dn['tmp'] = os.path.join(dn['sandbox'], 'tmp') for directory in ['checkout', 'install', 'tmp', 'baserockdir']: os.makedirs(dn[directory]) - dn['log'] = os.path.join(config.config['artifacts'], + dn['log'] = os.path.join(app.config['artifacts'], dn['cache'] + '.build-log') - if config.config.get('instances'): - dn['log'] += '.' + str(config.config.get('fork', 0)) + if app.config.get('instances'): + dn['log'] += '.' + str(app.config.get('fork', 0)) assembly_dir = dn['sandbox'] for directory in ['dev', 'tmp']: call(['mkdir', '-p', os.path.join(assembly_dir, directory)]) @@ -112,7 +112,7 @@ def run_sandboxed(dn, command, env=None, allow_parallel=False): if dn.get('build-mode') == 'bootstrap': # bootstrap mode: builds have some access to the host system, so they # can use the compilers etc. - tmpdir = config.config.get("TMPDIR", "/tmp") + tmpdir = app.config.get("TMPDIR", "/tmp") writable_paths = [dn['checkout'], dn['install'], tmpdir, ] @@ -204,7 +204,7 @@ def run_logged(dn, cmd_list): def run_extension(dn, deployment, step, method): log(dn, 'Running %s extension:' % step, method) extensions = utils.find_extensions() - tempfile.tempdir = config.config['tmp'] + tempfile.tempdir = app.config['tmp'] cmd_tmp = tempfile.NamedTemporaryFile(delete=False) cmd_bin = extensions[step][method] @@ -212,9 +212,9 @@ def run_extension(dn, deployment, step, method): if 'PYTHONPATH' in os.environ: envlist.append('PYTHONPATH=%s:%s' % (os.environ['PYTHONPATH'], - config.config['extsdir'])) + app.config['extsdir'])) else: - envlist.append('PYTHONPATH=%s' % config.config['extsdir']) + envlist.append('PYTHONPATH=%s' % app.config['extsdir']) for key, value in deployment.items(): if key.isupper(): @@ -229,7 +229,7 @@ def run_extension(dn, deployment, step, method): command.append(deployment.get('location') or deployment.get('upgrade-location')) - with app.chdir(config.config['defdir']): + with app.chdir(app.config['defdir']): try: with open(cmd_bin, "r") as infh: shutil.copyfileobj(infh, cmd_tmp) @@ -245,13 +245,13 @@ def run_extension(dn, deployment, step, method): def ccache_mounts(dn, ccache_target): - if config.config['no-ccache'] or 'repo' not in dn: + if app.config['no-ccache'] or 'repo' not in dn: mounts = [] else: name = os.path.basename(get_repo_url(dn['repo'])) if name.endswith('.git'): name = name[:-4] - ccache_dir = os.path.join(config.config['ccache_dir'], name) + ccache_dir = os.path.join(app.config['ccache_dir'], name) if not os.path.isdir(ccache_dir): os.mkdir(ccache_dir) @@ -263,7 +263,7 @@ def env_vars_for_build(dn): env = {} extra_path = [] - if config.config['no-ccache']: + if app.config['no-ccache']: ccache_path = [] else: ccache_path = ['/usr/lib/ccache'] @@ -272,7 +272,7 @@ def env_vars_for_build(dn): f for f in ('/baserock/binutils.meta', '/baserock/eglibc.meta', '/baserock/gcc.meta') if os.path.exists(f)) - if not config.config.get('no-distcc'): + if not app.config.get('no-distcc'): env['CCACHE_PREFIX'] = 'distcc' prefixes = [] @@ -289,15 +289,15 @@ def env_vars_for_build(dn): if dn.get('build-mode') == 'bootstrap': rel_path = extra_path + ccache_path full_path = [os.path.normpath(dn['sandbox'] + p) for p in rel_path] - path = full_path + config.config['base-path'] + path = full_path + app.config['base-path'] env['DESTDIR'] = dn.get('install') else: - path = extra_path + ccache_path + config.config['base-path'] + path = extra_path + ccache_path + app.config['base-path'] env['DESTDIR'] = os.path.join('/', os.path.basename(dn.get('install'))) env['PATH'] = ':'.join(path) env['PREFIX'] = dn.get('prefix') or '/usr' - env['MAKEFLAGS'] = '-j%s' % (dn.get('max-jobs') or config.config['max-jobs']) + env['MAKEFLAGS'] = '-j%s' % (dn.get('max-jobs') or app.config['max-jobs']) env['TERM'] = 'dumb' env['SHELL'] = '/bin/sh' env['USER'] = env['USERNAME'] = env['LOGNAME'] = 'tomjon' @@ -305,8 +305,8 @@ def env_vars_for_build(dn): env['HOME'] = '/tmp' env['TZ'] = 'UTC' - arch = config.config['arch'] - cpu = config.config['cpu'] + arch = app.config['arch'] + cpu = app.config['cpu'] abi = '' if arch.startswith(('armv7', 'armv5')): abi = 'eabi' @@ -315,8 +315,8 @@ def env_vars_for_build(dn): env['TARGET'] = cpu + '-baserock-linux-gnu' + abi env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi env['MORPH_ARCH'] = arch - env['DEFINITIONS_REF'] = config.config['def-version'] - env['PROGRAM_REF'] = config.config['my-version'] + env['DEFINITIONS_REF'] = app.config['def-version'] + env['PROGRAM_REF'] = app.config['my-version'] if dn.get('SOURCE_DATE_EPOCH'): env['SOURCE_DATE_EPOCH'] = dn['SOURCE_DATE_EPOCH'] diff --git a/ybd/utils.py b/ybd/utils.py index ee57aaf..62d3910 100644 --- a/ybd/utils.py +++ b/ybd/utils.py @@ -25,7 +25,6 @@ import sys from fs.osfs import OSFS from fs.multifs import MultiFS import calendar -from ybd.config import config # The magic number for timestamps: 2011-11-11 11:11:11 default_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) @@ -179,7 +178,7 @@ def _process_tree(root, srcpath, destpath, actionfunc): import re path = re.search('/.*$', re.search('tmp[^/]+/.*$', destpath).group(0)).group(0) - config.config['new-overlaps'] += [path] + app.config['new-overlaps'] += [path] try: os.unlink(destpath) except: @@ -393,7 +392,7 @@ def _find_extensions(paths): def find_extensions(): '''Scan definitions for extensions.''' - paths = [config.config['extsdir']] + paths = [app.config['extsdir']] return _find_extensions(paths) |