diff options
-rwxr-xr-x | kbas/__main__.py | 54 | ||||
-rw-r--r-- | kbas/config/kbas.conf | 6 | ||||
-rw-r--r-- | ybd/utils.py | 21 |
3 files changed, 74 insertions, 7 deletions
diff --git a/kbas/__main__.py b/kbas/__main__.py index 80de7ef..383a00a 100755 --- a/kbas/__main__.py +++ b/kbas/__main__.py @@ -26,6 +26,7 @@ from bottle import Bottle, request, response, template, static_file from subprocess import call, check_output from ybd import app, cache +from ybd.utils import cull_directory, get_free, sorted_ls bottle = Bottle() @@ -43,6 +44,7 @@ class KeyedBinaryArtifactServer(object): app.config['start-time'] = datetime.now() app.config['last-upload'] = datetime.now() app.config['downloads'] = 0 + app.config['tmp'] = app.config.get('tmp', app.config['artifact-dir']) try: import cherrypy @@ -91,6 +93,7 @@ class KeyedBinaryArtifactServer(object): @bottle.get('/get/<cache_id>') def get_artifact(cache_id): f = os.path.join(cache_id, cache_id) + call(['touch', f]) app.config['downloads'] += 1 return static_file(f, root=app.config['artifact-dir'], download=True, mimetype='application/x-tar') @@ -98,9 +101,8 @@ class KeyedBinaryArtifactServer(object): @bottle.get('/') @bottle.get('/status') def status(): - stat = os.statvfs(app.config['artifact-dir']) - free = stat.f_frsize * stat.f_bavail / 1000000000 - artifacts = len(os.listdir(app.config['artifact-dir'])) + artifact_dir = app.config['artifact-dir'] + artifacts = len(os.listdir(artifact_dir)) started = app.config['start-time'].strftime('%y-%m-%d %H:%M:%S') downloads = app.config['downloads'] last_upload = app.config['last-upload'].strftime('%y-%m-%d %H:%M:%S') @@ -108,7 +110,8 @@ class KeyedBinaryArtifactServer(object): content += [['Last upload:', last_upload, None]] if app.config.get('last-reject'): content += [['Last reject:', app.config['last-reject'], None]] - content += [['Space:', str(free) + 'GB', None]] + content += [['Artifact directory:', artifact_dir, None]] + content += [['Space:', str(get_free(artifact_dir)) + 'GB', None]] content += [['Artifacts:', str(artifacts), None]] content += [['Downloads:', downloads, None]] return template('kbas', @@ -116,6 +119,34 @@ class KeyedBinaryArtifactServer(object): content=content, css='static/style.css') + @bottle.get('/cull/') + @bottle.get('/cull/<space>') + def cull(space=""): + artifact_dir = app.config['artifact-dir'] + try: + target = int(space) + if target in range(app.config.get('min-cull', 0), + app.config.get('max-cull', 0) + 1): + + deleted = cull_directory(artifact_dir, target) + free = str(get_free(artifact_dir)) + app.log('CULL', 'Culled %s items, %sGB free space in' + % (deleted, free), artifact_dir) + + content = [['Culled files: ', deleted, None]] + return template('kbas', + title='KBAS Cull Status', + content=content, + css='/static/style.css') + except ValueError: + pass + + content = [['No cull with:', space, None]] + return template('kbas', + title='KBAS Cull Status', + content=content, + css='/static/style.css') + @bottle.post('/upload') def post_artifact(): if app.config['password'] is 'insecure' or \ @@ -138,7 +169,8 @@ class KeyedBinaryArtifactServer(object): response.status = 405 # not allowed, this artifact exists return - tempfile.tempdir = app.config['artifact-dir'] + artifact_dir = app.config['artifact-dir'] + tempfile.tempdir = artifact_dir tmpdir = tempfile.mkdtemp() try: upload = request.files.get('file') @@ -164,10 +196,18 @@ class KeyedBinaryArtifactServer(object): checksum = cache.md5(artifact) with open(artifact + '.md5', "a") as f: f.write(checksum) - shutil.move(tmpdir, os.path.join(app.config['artifact-dir'], - cache_id)) + + shutil.move(tmpdir, os.path.join(artifact_dir, cache_id)) response.status = 201 # success! app.config['last-upload'] = datetime.now() + + if os.fork() == 0: + deleted = cull_directory(artifact_dir, app.config['min-cull']) + if deleted > 0: + app.log('CULL', 'Culled %s items, %sGB free space in' + % (deleted, get_free(artifact_dir)), artifact_dir) + os._exit(0) + return except: # something went wrong, clean up diff --git a/kbas/config/kbas.conf b/kbas/config/kbas.conf index 02b0b70..5a219f2 100644 --- a/kbas/config/kbas.conf +++ b/kbas/config/kbas.conf @@ -45,6 +45,12 @@ artifact-dir: '/src/artifacts' # cases... '0.0.0.0 means all IPv4 addresses on the local machine' host: 0.0.0.0 +# max-cull sets the maximum space the cull algorithm will accept (in GB) +# min-cull sets the minimum space the cull algorithm will accept (in GB) +# by default the cull will try to ensure min-cull GB is available +max-cull: 10 +min-cull: 5 + # port to serve on port: 8000 diff --git a/ybd/utils.py b/ybd/utils.py index 9ee321e..d5b5311 100644 --- a/ybd/utils.py +++ b/ybd/utils.py @@ -435,6 +435,27 @@ def sorted_ls(path): return list(sorted(os.listdir(path), key=mtime)) +def cull_directory(artifact_dir, target_space): + tempfile.tempdir = artifact_dir + artifacts = sorted_ls(artifact_dir) + deleted = 0 + for artifact in artifacts: + if get_free(artifact_dir) < target_space: + path = os.path.join(artifact_dir, artifact) + if os.path.exists(path): + tmpdir = tempfile.mkdtemp() + shutil.move(path, os.path.join(tmpdir, 'to-delete')) + app.remove_dir(tmpdir) + deleted += 1 + return deleted + + +def get_free(directory): + # calculate free space in GB + stat = os.statvfs(directory) + return stat.f_frsize * stat.f_bavail / 1073741824 + + @contextlib.contextmanager def monkeypatch(obj, attr, new_value): '''Temporarily override the attribute of some object. |