Add CI testing for raw files

Created two tests for the creation of appropriate local git repos
from folders of raw files.

1. One raw file. Verify that the desired form of the .lorry file creates
a corresponding git repo with the correct relative paths.

2. Multiple raw files. Verify all the above, plus the additon of all raw
files and the updating of the realtive paths upon changes of the .lorry
file.

7 files changed, 205 insertions, 1 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fd377a2..970c34f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,8 +21,12 @@ test:
     # Deps for running tests
     - apt-get install -y -qq cmdtest
 
+    # Download git-lfs repo
+    - apt-get install -y -qq curl
+    - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
     # Deps to run lorry tests
-    - apt-get install -y -qq git bzr cvs mercurial subversion hg-fast-export git-svn git-cvs bzr-fastimport
+    - apt-get install -y -qq git bzr cvs mercurial subversion hg-fast-export git-svn git-cvs git-lfs bzr-fastimport
     - pip3 install pyyaml
     - pip3 install https://gitlab.com/trovekube/cliapp/-/archive/cliapp-1.20180812.1/cliapp-cliapp-1.20180812.1.tar.gz
 
diff --git a/tests/raw-multiple-files.script b/tests/raw-multiple-files.script
new file mode 100755
index 0000000..4c16c51
--- /dev/null
+++ b/tests/raw-multiple-files.script
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Tests converting a folder with three raw files into a git repository.
+# Verifies that updating the relative paths in the .lorry file and
+# running again updates the relative paths in the git repo.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/raw-multiple-files.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-multiple-files.lorry" > /dev/null
+
+# update the lorry file, verify relative paths in git repo have changed
+sed -i 's/subdir_two/./g' ${DATADIR}/raw-multiple-files.lorry
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-multiple-files.lorry" > /dev/null
+
+# verify that the git repositories were created successfully
+cd "$workdir/multiple-files-repo/git-a"
+
+# verify the files were created correctly the first time
+git ls-tree -r --name-only HEAD
+
+cd "$workdir/multiple-files-repo/git-b/"
+
+# verify the files have been updated correctly
+git ls-tree -r --name-only HEAD
+
+# list the commit messages
+git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
diff --git a/tests/raw-multiple-files.setup b/tests/raw-multiple-files.setup
new file mode 100755
index 0000000..7e4aa12
--- /dev/null
+++ b/tests/raw-multiple-files.setup
@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Creates three raw files, and a lorry file specifying their downstream
+# subdirectories.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+# create the raw file directory
+repo="$DATADIR/raw-multiple-files-repo"
+mkdir "$repo"
+echo "first line, test 1" > "$repo/test_1.txt"
+echo "first line, test 2" > "$repo/test_2.txt"
+echo "first line, test 3" > "$repo/test_3.txt"
+
+# create the .lorry file for the raw file
+cat <<EOF > $DATADIR/raw-multiple-files.lorry
+{
+    "multiple-files-repo": {
+        "type": "raw-file",
+        "urls": [
+            {
+                "url": "file://$repo/test_1.txt",
+                "destination": "subdir_one"
+            },
+            {
+                "url": "file://$repo/test_2.txt",
+                "destination": "subdir_one"
+            },
+            {
+                "url": "file://$repo/test_3.txt",
+                "destination": "subdir_two"
+            }
+        ]
+    }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/raw-multiple-files.stdout b/tests/raw-multiple-files.stdout
new file mode 100644
index 0000000..1a55aa5
--- /dev/null
+++ b/tests/raw-multiple-files.stdout
@@ -0,0 +1,14 @@
+.gitattributes
+subdir_one/test_1.txt
+subdir_one/test_2.txt
+subdir_two/test_3.txt
+.gitattributes
+subdir_one/test_1.txt
+subdir_one/test_2.txt
+test_3.txt
+Remove redundant file paths.
+import test_3.txt
+import test_3.txt
+import test_2.txt
+import test_1.txt
+Ensure LFS is configured
diff --git a/tests/raw-single-file.script b/tests/raw-single-file.script
new file mode 100755
index 0000000..b46e0c1
--- /dev/null
+++ b/tests/raw-single-file.script
@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# Tests converting a folder with one raw file to a git repository.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/raw-single-file.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-single-file.lorry" > /dev/null
+
+# verify that the git repository was created successfully
+cd "$workdir/single-file-repo/git-a/"
+
+# verify the file has been added correctly
+git ls-tree -r --name-only HEAD
+
+# list the commit messages
+git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
diff --git a/tests/raw-single-file.setup b/tests/raw-single-file.setup
new file mode 100755
index 0000000..dde24c1
--- /dev/null
+++ b/tests/raw-single-file.setup
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Creates a raw file and corresponding lorry file.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+# create the raw file directory
+repo="$DATADIR/raw-single-file-repo"
+mkdir "$repo"
+echo "first line" > "$repo/test.txt"
+
+# create the .lorry file for the raw file
+cat <<EOF > $DATADIR/raw-single-file.lorry
+{
+    "single-file-repo": {
+        "type": "raw-file",
+        "urls": [
+            {
+                "url": "file://$repo/test.txt",
+                "destination": "subdir_test"
+            }
+        ]
+    }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/raw-single-file.stdout b/tests/raw-single-file.stdout
new file mode 100644
index 0000000..fe73f17
--- /dev/null
+++ b/tests/raw-single-file.stdout
@@ -0,0 +1,4 @@
+.gitattributes
+subdir_test/test.txt
+import test.txt
+Ensure LFS is configured