debuginfod server: support .deb/.ddeb archives

Add support for scanning .deb / .ddeb files, enabled with a new
command line option "-U".  Using a synthetic .deb/.ddeb from a Ubuntu
18 machine, extend the debuginfod testsuite with some .deb processing,
if the dpkg-deb binary is installed.

Signed-off-by: Frank Ch. Eigler <fche@redhat.com>

8 files changed, 77 insertions, 23 deletions

diff --git a/tests/ChangeLog b/tests/ChangeLog
index 7103cf51..02a8f75f 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,10 @@
+2019-12-22  Frank Ch. Eigler  <fche@redhat.com>
+
+	* debuginfod-debs/*: New test files, based on
+	https://wiki.debian.org/Packaging/Intro.
+	* run-debuginfod-find.sh: Test deb file processing (if dpkg
+	installed).
+
 2019-12-04  Frank Ch. Eigler  <fche@redhat.com>
 
 	* run-debuinfod-find.sh: Test $DEBUGINFOD_PROGRESS.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2c3ac299..21b25986 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -462,7 +462,13 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
 	     debuginfod-rpms/rhel7/hello2-1.0-2.src.rpm \
 	     debuginfod-rpms/rhel7/hello2-1.0-2.x86_64.rpm \
 	     debuginfod-rpms/rhel7/hello2-debuginfo-1.0-2.x86_64.rpm \
-	     debuginfod-rpms/rhel7/hello2-two-1.0-2.x86_64.rpm
+	     debuginfod-rpms/rhel7/hello2-two-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/rhel7/hello2-two-1.0-2.x86_64.rpm \
+	     debuginfod-debs/hithere-dbgsym_1.0-1_amd64.ddeb \
+	     debuginfod-debs/hithere_1.0-1.debian.tar.xz \
+	     debuginfod-debs/hithere_1.0-1.dsc \
+	     debuginfod-debs/hithere_1.0-1_amd64.deb \
+	     debuginfod-debs/hithere_1.0.orig.tar.gz
 
 
 
diff --git a/tests/debuginfod-debs/hithere-dbgsym_1.0-1_amd64.ddeb b/tests/debuginfod-debs/hithere-dbgsym_1.0-1_amd64.ddeb
new file mode 100644
index 00000000..f9879eb6
--- /dev/null
+++ b/tests/debuginfod-debs/hithere-dbgsym_1.0-1_amd64.ddeb
diff --git a/tests/debuginfod-debs/hithere_1.0-1.debian.tar.xz b/tests/debuginfod-debs/hithere_1.0-1.debian.tar.xz
new file mode 100644
index 00000000..9f0ce684
--- /dev/null
+++ b/tests/debuginfod-debs/hithere_1.0-1.debian.tar.xz
diff --git a/tests/debuginfod-debs/hithere_1.0-1.dsc b/tests/debuginfod-debs/hithere_1.0-1.dsc
new file mode 100644
index 00000000..d5f72b90
--- /dev/null
+++ b/tests/debuginfod-debs/hithere_1.0-1.dsc
@@ -0,0 +1,19 @@
+Format: 3.0 (quilt)
+Source: hithere
+Binary: hithere
+Architecture: any
+Version: 1.0-1
+Maintainer: Lars Wirzenius <liw@liw.fi>
+Standards-Version: 3.9.2
+Build-Depends: debhelper (>= 9)
+Package-List:
+ hithere deb misc optional arch=any
+Checksums-Sha1:
+ 2dcd65497a12a3ea03223f52186447bd5733dce9 617 hithere_1.0.orig.tar.gz
+ 0b71331ef1c714c5bac67878551864b7356c56ce 764 hithere_1.0-1.debian.tar.xz
+Checksums-Sha256:
+ 63062b582a712f169f37a5f52a41aa3ca9a405aafb8aa837bc906fa413b62cdb 617 hithere_1.0.orig.tar.gz
+ 9afa907e360e626639ccb86b86e799429bea27149034aec5d5c7e500971d651e 764 hithere_1.0-1.debian.tar.xz
+Files:
+ 5b2830fa1fcd44ce489774771625526e 617 hithere_1.0.orig.tar.gz
+ 70106164d9397c70c2c1a4594e9897e4 764 hithere_1.0-1.debian.tar.xz
diff --git a/tests/debuginfod-debs/hithere_1.0-1_amd64.deb b/tests/debuginfod-debs/hithere_1.0-1_amd64.deb
new file mode 100644
index 00000000..11d1e958
--- /dev/null
+++ b/tests/debuginfod-debs/hithere_1.0-1_amd64.deb
diff --git a/tests/debuginfod-debs/hithere_1.0.orig.tar.gz b/tests/debuginfod-debs/hithere_1.0.orig.tar.gz
new file mode 100644
index 00000000..23abea70
--- /dev/null
+++ b/tests/debuginfod-debs/hithere_1.0.orig.tar.gz
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index 4cf61389..90dafe00 100755
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -18,6 +18,10 @@
 
 . $srcdir/test-subr.sh  # includes set -e
 
+# for test case debugging, uncomment:
+# set -x
+# VERBOSE=-vvvv
+
 DB=${PWD}/.debuginfod_tmp.sqlite
 tempfiles $DB
 export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache
@@ -30,7 +34,7 @@ cleanup()
   if [ $PID1 -ne 0 ]; then kill $PID1; wait $PID1; fi
   if [ $PID2 -ne 0 ]; then kill $PID2; wait $PID2; fi
 
-  rm -rf F R L ${PWD}/.client_cache*
+  rm -rf F R D L ${PWD}/.client_cache*
   exit_cleanup
 }
 
@@ -52,8 +56,8 @@ done
 # So we gather the LD_LIBRARY_PATH with this cunning trick:
 ldpath=`testrun sh -c 'echo $LD_LIBRARY_PATH'`
 
-mkdir F R L
-# not tempfiles F R L - they are directories which we clean up manually
+mkdir F R L D
+# not tempfiles F R L D - they are directories which we clean up manually
 ln -s ${abs_builddir}/dwfllines L/foo   # any program not used elsewhere in this test
 
 wait_ready()
@@ -77,12 +81,13 @@ wait_ready()
   done;
 
   if [ $timeout -eq 0 ]; then
-    echo "metric $what never changed to $value on port $port"
+      echo "metric $what never changed to $value on port $port"
+      curl -s http://127.0.0.1:$port/metrics
     exit 1;
   fi
 }
 
-env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod -F -R -d $DB -p $PORT1 -t0 -g0 R F L &
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p $PORT1 -t0 -g0 R F L &
 PID1=$!
 # Server must become ready
 wait_ready $PORT1 'ready' 1
@@ -161,11 +166,11 @@ tempfiles vlog2
 filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID2 ${PWD}/prog2.c`
 cmp $filename ${PWD}/prog2.c
 
-cp -rp ${abs_srcdir}/debuginfod-rpms R
+cp -rvp ${abs_srcdir}/debuginfod-rpms R
 kill -USR1 $PID1
 # All rpms need to be in the index
 rpms=$(find R -name \*rpm | wc -l)
-wait_ready $PORT1 'scanned_total{source="rpm"}' $rpms
+wait_ready $PORT1 'scanned_total{source=".rpm archive"}' $rpms
 
 kill -USR1 $PID1  # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs
 # Expect all source files found in the rpms (they are all called hello.c :)
@@ -180,7 +185,7 @@ for i in $newrpms; do
     mkdir $subdir;
     cd $subdir;
     ls -lah ../$i
-    rpm2cpio ../$i | cpio -id;
+    rpm2cpio ../$i | cpio -ivd;
     cd ..;
 done
 sourcefiles=$(find -name \*\\.debug \
@@ -190,11 +195,11 @@ sourcefiles=$(find -name \*\\.debug \
 cd ..
 rm -rf extracted
 
-wait_ready $PORT1 'found_sourcerefs_total{source="rpm"}' $sourcefiles
+wait_ready $PORT1 'found_sourcerefs_total{source=".rpm archive"}' $sourcefiles
 
-# Run a bank of queries against the debuginfod-rpms test cases
+# Run a bank of queries against the debuginfod-rpms / debuginfod-debs test cases
 
-rpm_test() {
+archive_test() {
     __BUILDID=$1
     __SOURCEPATH=$2
     __SOURCESHA1=$3
@@ -208,24 +213,26 @@ rpm_test() {
     buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
              -a $filename | grep 'Build ID' | cut -d ' ' -f 7`
     test $__BUILDID = $buildid
-    
-    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH`
-    hash=`cat $filename | sha1sum | awk '{print $1}'`
-    test $__SOURCESHA1 = $hash
+
+    if test "x$__SOURCEPATH" != "x"; then
+        filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH`
+        hash=`cat $filename | sha1sum | awk '{print $1}'`
+        test $__SOURCESHA1 = $hash
+    fi
 }
 
 
 # common source file sha1
 SHA=f4a1a8062be998ae93b8f1cd744a398c6de6dbb1
 # fedora30
-rpm_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
-rpm_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
+archive_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
+archive_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
 # rhel7
-rpm_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA
-rpm_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA
+archive_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA
+archive_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA
 # rhel6
-rpm_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA
-rpm_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA
+archive_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA
+archive_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA
 
 RPM_BUILDID=d44d42cbd7d915bc938c81333a21e355a6022fb7 # in rhel6/ subdir, for a later test
 
@@ -260,13 +267,28 @@ export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache2
 mkdir -p $DEBUGINFOD_CACHE_PATH
 # NB: inherits the DEBUGINFOD_URLS to the first server
 # NB: run in -L symlink-following mode for the L subdir
-env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod -F -d ${DB}_2 -p $PORT2 -L L &
+env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -U -d ${DB}_2 -p $PORT2 -L L D &
 PID2=$!
 tempfiles ${DB}_2
 wait_ready $PORT2 'ready' 1
 
 # have clients contact the new server
 export DEBUGINFOD_URLS=http://127.0.0.1:$PORT2
+
+if type dpkg-deb 2>/dev/null; then
+    # copy in the deb files
+    cp -rvp ${abs_srcdir}/debuginfod-debs/*deb D
+    kill -USR1 $PID2
+    # All debs need to be in the index
+    debs=$(find D -name \*.deb | wc -l)
+    wait_ready $PORT2 'scanned_total{source=".deb archive"}' `expr $debs`
+    ddebs=$(find D -name \*.ddeb | wc -l)
+    wait_ready $PORT2 'scanned_total{source=".ddeb archive"}' `expr $ddebs`
+
+    # ubuntu
+    archive_test f17a29b5a25bd4960531d82aa6b07c8abe84fa66 "" ""
+fi
+
 rm -rf $DEBUGINFOD_CACHE_PATH
 testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID