yarns: Make the distbuild yarn expose the worker's artifact cache over HTTP

OSTree can easily pull over HTTP. All that is necessary is to expose the
repo directory. This commit adds a simple HTTP server to do this in the
test suite. Actual implementations should use something better, like
lighttpd.

Also add some logging of the cache servers in this yarn to help debug.

Change-Id: Ic65390f550c972c5f1072bbca8d80e8b56723158

3 files changed, 32 insertions, 5 deletions

diff --git a/ostree-repo-server b/ostree-repo-server
new file mode 100755
index 00000000..e6dc4a56
--- /dev/null
+++ b/ostree-repo-server
@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler 
+from SocketServer import ThreadingMixIn
+
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+    """Handle requests in a separate thread"""
+
+handler = SimpleHTTPRequestHandler
+handler.protocol_version="HTTP/1.0"
+server_address = ('', 12324)
+
+httpd = ThreadedHTTPServer(server_address, handler)
+httpd.serve_forever()
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index faae44a8..7a4a8484 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -1061,13 +1061,14 @@ Distbuild
     read_cache_server_pid_file="$DATADIR/read-cache-server-pid"
     start_cache_server "$read_cache_server_port_file" \
                        "$read_cache_server_pid_file" \
-                       "$artifact_dir"
+                       "$artifact_dir" "$DATADIR/communal-cache.log"
 
     write_cache_server_port_file="$DATADIR/write-cache-server-port"
     write_cache_server_pid_file="$DATADIR/write-cache-server-pid"
     start_cache_server "$write_cache_server_port_file" \
                        "$write_cache_server_pid_file" \
-                       "$artifact_dir" --enable-writes
+                       "$artifact_dir" "$DATADIR/writeable-cache.log" \
+                       --enable-writes
 
     IMPLEMENTS FINALLY the communal cache server is terminated
     stop_daemon "$DATADIR/read-cache-server-pid"
@@ -1082,7 +1083,7 @@ Distbuild
     worker_cache_pid_file="$DATADIR/worker-cache-server-pid"
     start_cache_server "$worker_cache_port_file" \
                        "$worker_cache_pid_file" \
-                       "$worker_artifacts"
+                       "$worker_artifacts" "$DATADIR/worker-cache.log"
 
     # start worker daemon
     worker_daemon_port_file="$DATADIR/worker-daemon-port"
@@ -1104,6 +1105,14 @@ Distbuild
     rm "$worker_daemon_port_file"
     echo "$worker_daemon_port" >"$worker_daemon_port_file"
 
+    # serve artifact cache over http
+    worker_repo_pid_file="$DATADIR/worker-repo-pid"
+    mkdir "$worker_artifacts/repo"
+    cd "$worker_artifacts/repo"
+    start-stop-daemon --start --pidfile="$worker_repo_pid_file" \
+        --background --make-pidfile --verbose \
+        --startas="$SRCDIR/ostree-repo-server"
+
     # start worker helper
     helper_pid_file="$DATADIR/worker-daemon-helper-pid"
     start-stop-daemon --start --pidfile="$helper_pid_file" \
@@ -1118,13 +1127,15 @@ Distbuild
     exec "$SRCDIR/morph" --quiet \
         --cachedir-min-space=0 --tempdir-min-space=0 \
         --no-default-config --config "$DATADIR/morph.conf" \
-        --cachedir "$DATADIR/distbuild-worker-cache" "$@"
+        --cachedir "$DATADIR/distbuild-worker-cache" \
+        --ostree-repo-mode="archive_z2" "$@"
     EOF
 
     IMPLEMENTS FINALLY the distbuild worker is terminated
     stop_daemon "$DATADIR/worker-cache-server-pid"
     stop_daemon "$DATADIR/worker-daemon-pid"
     stop_daemon "$DATADIR/worker-daemon-helper-pid"
+    stop_daemon "$DATADIR/worker-repo-pid"
 
     IMPLEMENTS GIVEN a distbuild controller
     worker_cache_port_file="$DATADIR/worker-cache-server-port"
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index e7011091..3252b30f 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -39,7 +39,7 @@ run_morph()
         "$SRCDIR"/morph --verbose \
             --cachedir-min-space=0 --tempdir-min-space=0 \
             --no-default-config --config "$DATADIR/morph.conf" \
-            --log="$DATADIR/log-$1"  \
+            --log="$DATADIR/log-$1" --ostree-repo-mode="archive_z2" \
             "$@" 2> "$DATADIR/result-$1" > "$DATADIR/out-$1"
         local exit_code="$?"
         for o in log result out; do
@@ -194,6 +194,7 @@ start_cache_server(){
         --background --make-pidfile --verbose \
         --startas="$SRCDIR/morph-cache-server" -- \
         --port-file="$1" --no-fcgi \
+        --log="$4" \
         --repo-dir="$DATADIR/gits" --direct-mode \
         --bundle-dir="$DATADIR/bundles" \
         --artifact-dir="$3" "$@"