Starting to add in some tests for OSTree.

Had to wrap up OSTree creation in a bash script and serve using a lightweight python http server

7 files changed, 251 insertions, 23 deletions

diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py
index 471d7bd41..c175505a7 100644
--- a/buildstream/plugins/sources/ostree.py
+++ b/buildstream/plugins/sources/ostree.py
@@ -22,44 +22,45 @@
 
 """
 
-import os
-import subprocess
 from gi.repository import OSTree, Gio
-
-from buildstream import Source, SourceError, ProgramNotFoundError
-from buildstream import utils
+from buildstream import Source, LoadError
 
 
 class OSTreeSource(Source):
 
     def configure(self, node):
+        project = self.get_project()
+
         self.remote_name = "origin"
-        self.url = utils.node_get_member(node, str, 'url')
-        self.ref = utils.node_get_member(node, str, 'ref', '')
-        self.branch = utils.node_get_member(node, str, 'branch', '')
+        self.url = project.translate_url(self.node_get_member(node, str, 'url'))
+        self.ref = self.node_get_member(node, str, 'ref')
+        self.track = self.node_get_member(node, str, 'track', '')
 
         # (optional) Not all repos are signed. But if they are, get the gpg key
-        self.gpg_key = utils.node_get_member(node, str, 'gpg_key', None)
+        try:
+            self.gpg_key = self.node_get_member(node, str, 'gpg_key', None)
+        except LoadError:
+            self.gpg_key = None
 
         self.ostree_dir = "repo"    # Assume to be some tmp dir
 
     def preflight(self):
-        # Check if OSTree is installed, get the binary at the same time
-        # TODO Actually, this isn't needed due to python bindings?
-        try:
-            self.host_ostree = utils.get_host_tool("ostree")
-        except ProgramNotFoundError as e:
-            raise SourceError("Prerequisite programs not found in host environment for OSTree", e)
+        return
 
     def get_unique_key(self):
         return [self.url, self.ref]
 
     def refresh(self, node):
-        # Not sure what to put here
+        # Not sure what else to put here
 
         self.load_ostree(self.ostree_dir)
         self.fetch_ostree(self.remote_name, self.ref)
 
+        # TODO Only return true if things have been updated. Not sure
+        # how I'd do this with OSTree. Surely the ref used means nothing
+        # is different, unless this has not been pulled yet.
+        return True
+
     def fetch(self):
         # Pull the OSTree from the remote
 
@@ -72,6 +73,9 @@ class OSTreeSource(Source):
         self.checkout_ostree(directory, self.ref)
         pass
 
+    def consistent(self):
+        return True
+
     ###########################################################
     #                     Local Functions                     #
     ###########################################################
@@ -81,7 +85,7 @@ class OSTreeSource(Source):
         # ostree --repo=repo init --mode=archive-z2
 
         self.ost = OSTree.Repo.new(Gio.File.new_for_path(repo_dir))
-        self.ost.create(OSTree.RepoMode.ARCHIVE_Z2 , None)
+        self.ost.create(OSTree.RepoMode.ARCHIVE_Z2, None)
 
     def load_ostree(self, repo_dir):
         # Loads an existing OSTree repo from the given `repo_dir`
@@ -92,7 +96,7 @@ class OSTreeSource(Source):
     def fetch_ostree(self, remote, ref):
         # ostree --repo=repo pull --mirror freedesktop:runtime/org.freedesktop.Sdk/x86_64/1.4
 
-        progress = None  # Alternatively OSTree.AsyncProgress
+        progress = None  # Alternatively OSTree.AsyncProgress, None assumed to block
         cancellable = None  # Alternatively Gio.Cancellable
 
         self.ost.pull(remote, ref, OSTree.RepoPullFlags.MIRROR, progress, cancellable)
@@ -117,18 +121,18 @@ class OSTreeSource(Source):
         self.ost.remote_gpg_import(name, stream, None, 0, None)
         return
 
-    def ls_branches(self):
-        # Grab the named refs/branches that exist in this repo
+    def ls_tracks(self):
+        # Grab the named refs/tracks that exist in this repo
         # ostree --repo=repo refs
         _, refs = self.ost.list_refs()
 
         # Returns a dict of {branch: head-ref}
         return refs.keys()
 
-    def branch_head(self, branch_name):
-        # Get the checksum of the head commit of the branch `branch_name`
+    def track_head(self, track_name):
+        # Get the checksum of the head commit of the branch `track_name`
         # ostree --repo=repo log runtime/org.freedesktop.Sdk/x86_64/1.4
-        _, head_ref = self.ost.resolve_ref(branch_name)
+        _, head_ref = self.ost.resolve_ref(track_name)
 
         return head_ref
 
diff --git a/tests/sources/ostree.py b/tests/sources/ostree.py
new file mode 100644
index 000000000..00fad5654
--- /dev/null
+++ b/tests/sources/ostree.py
@@ -0,0 +1,179 @@
+import os
+import pytest
+import tempfile
+import subprocess
+import re
+
+from buildstream import SourceError
+from buildstream import utils
+
+# import our common fixture
+from .fixture import Setup
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'ostree',
+)
+
+class OSTreeSetup(Setup):
+
+    def __init__(self, datafiles, tmpdir, bstfile=None):
+
+        if not bstfile:
+            bstfile = 'target.bst'
+
+        # This is where we'll put ostree checkouts
+        self.origin_dir = os.path.join(str(tmpdir), 'origin')
+        if not os.path.exists(self.origin_dir):
+            os.mkdir(self.origin_dir)
+
+        super().__init__(datafiles, bstfile, tmpdir)
+
+
+def run_ostree_bash_script():
+    # Run the generate-ostree.sh script
+
+    process = subprocess.Popen(
+        ['%s/generate-ostree.sh' % (DATA_DIR,)],
+        stderr=subprocess.PIPE)
+    process.wait()
+    BASH_DONE = True
+
+    return process.returncode
+
+
+def run_ostree_cli(repo, cmd):
+    if type(cmd) is not list:
+        cmd = [cmd]
+
+    arg = ['ostree', '--repo=%s' % (repo,)]
+    arg.extend(cmd)
+    process = subprocess.Popen(
+        arg,
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE)
+    out, err = process.communicate()
+
+    return process.returncode, out, err
+
+
+def run_ostree_mini_server():
+    import http.server
+    import socketserver
+    import threading
+
+    PORT = 8000
+    Handler = http.server.SimpleHTTPRequestHandler
+    httpd = socketserver.TCPServer(("", PORT), Handler)
+
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+
+###############################################################
+#                            Tests                            #
+###############################################################
+
+
+def test_ostree_shell_exe():
+    # Run the generate-ostree.sh script
+    # Does it run ok?
+    ret = run_ostree_bash_script()
+
+    assert ret == 0
+
+
+def test_ostree_shell_dir_exist(tmpdir):
+    # tmp/repo and tmp/files directories should exist
+
+    run_ostree_bash_script()
+
+    assert os.path.isdir("tmp/repo")
+    assert os.path.isdir("tmp/files")
+
+
+def test_ostree_shell_branches():
+    # only 'my-branch' should exist
+
+    run_ostree_bash_script()
+    exit, out, err = run_ostree_cli("tmp/repo", "refs")
+    assert out.decode('unicode-escape').strip() == "my-branch"
+
+    exit, out, err = run_ostree_cli("repofoo", "refs")
+    assert err.decode('unicode-escape') != ''
+
+
+def test_ostree_shell_commits():
+    # only 2 commits
+    global REF_HEAD, REF_NOTHEAD
+
+    run_ostree_bash_script()
+    exit, out, err = run_ostree_cli("tmp/repo", ["log", "my-branch"])
+
+    reg = re.compile(r'commit ([a-z0-9]{64})')
+    commits = [m.groups()[0] for m in reg.finditer(str(out))]
+    assert len(commits) == 2
+
+
+def test_ostree_shell_url_reachable():
+    # http://127.0.0.1:8000 returns 200_ok
+    import http
+    run_ostree_mini_server()
+    h = http.client.HTTPConnection('127.0.0.1:8000')
+    h.request("GET", '/')
+    res = h.getresponse()
+
+    assert res.code == 200
+
+###
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'head'))
+def test_ostree_conf(tmpdir, datafiles):
+
+    setup = OSTreeSetup(datafiles, tmpdir)
+    assert(setup.source.get_kind() == 'ostree')
+
+    # Test other config settings
+    assert(setup.source.remote_name == 'origin')
+    assert(setup.source.url == '127.0.0.1:8000')
+    assert(setup.source.track == 'my-branch')
+    assert(setup.source.gpg_key is None)
+    assert(setup.source.ostree_dir == 'repo')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'head'))
+def test_ostree_fetch(tmpdir, datafiles):
+    setup = OSTreeSetup(datafiles, tmpdir)
+    assert(setup.source.get_kind() == 'ostree')
+
+    # Make sure we preflight and fetch first, cant stage without fetching
+    setup.source.preflight()
+    setup.source.fetch()
+
+    # Check to see if the directory contains basic OSTree directories
+    expected = ['objects', 'config', 'tmp', 'extensions', 'state', 'refs']
+    indir = os.listdir(setup.source.ostree_dir)
+    assert(set(expected) <= set(indir))
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'head'))
+def test_ostree_stage(tmpdir, datafiles):
+    setup = OSTreeSetup(datafiles, tmpdir)
+    assert(setup.source.get_kind() == 'ostree')
+
+    # Make sure we preflight and fetch first, cant stage without fetching
+    setup.source.preflight()
+    setup.source.fetch()
+
+    # Stage the file and just check that it's there
+    stagedir = os.path.join(setup.context.builddir, 'repo')
+    setup.source.stage(stagedir)
+
+
+def test_ostree_head_configure():
+    #  ref: 85a4d86655f56715aea16170a0599218f8f42a8efea4727deb101b1520325f7e
+    pass
+
+
+def test_ostree_nothead_configure():
+    # ref: 3c11e7aed983ad03a2982c33f061908879033dadce4c21ce93243c118264ee0f
+    pass
diff --git a/tests/sources/ostree/generate-ostree.sh b/tests/sources/ostree/generate-ostree.sh
new file mode 100755
index 000000000..6deba9206
--- /dev/null
+++ b/tests/sources/ostree/generate-ostree.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+TMP_DIR=tmp
+
+REPO=`pwd`/$TMP_DIR/repo
+DATA_DIR=$TMP_DIR/files
+
+# Make sure this is only ran once
+if [ ! -d "$REPO" ]; then
+    mkdir -p $REPO
+    mkdir -p $DATA_DIR
+
+    ostree --repo=$REPO init
+
+    cd $DATA_DIR
+
+    # Do first commit
+    # 3c11e7aed983ad03a2982c33f061908879033dadce4c21ce93243c118264ee0f
+    echo "1" > foo
+    ostree --repo=$REPO commit --branch=my-branch --subject="Initial commit" --body="This is the first commit."
+
+    # Second commit
+    # 85a4d86655f56715aea16170a0599218f8f42a8efea4727deb101b1520325f7e
+    rm foo
+    echo "1" > bar
+    ostree --repo=$REPO commit --branch=my-branch --subject="Another commit" --body="Removing foo and adding bar"
+fi
\ No newline at end of file
diff --git a/tests/sources/ostree/head/project.conf b/tests/sources/ostree/head/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/ostree/head/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/ostree/head/target.bst b/tests/sources/ostree/head/target.bst
new file mode 100644
index 000000000..dc1611c4f
--- /dev/null
+++ b/tests/sources/ostree/head/target.bst
@@ -0,0 +1,7 @@
+kind: pony
+description: This is the pony
+sources:
+- kind: ostree
+  url: 127.0.0.1:8000
+  ref: 85a4d86655f56715aea16170a0599218f8f42a8efea4727deb101b1520325f7e
+  track: my-branch
\ No newline at end of file
diff --git a/tests/sources/ostree/nothead/project.conf b/tests/sources/ostree/nothead/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/ostree/nothead/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/ostree/nothead/target.bst b/tests/sources/ostree/nothead/target.bst
new file mode 100644
index 000000000..73f85d85f
--- /dev/null
+++ b/tests/sources/ostree/nothead/target.bst
@@ -0,0 +1,7 @@
+kind: pony
+description: This is the pony
+sources:
+- kind: ostree
+  url: 127.0.0.1:8000
+  ref: 3c11e7aed983ad03a2982c33f061908879033dadce4c21ce93243c118264ee0f
+  branch: my-branch
\ No newline at end of file