functional-tests: Add 302-miner-removable-devices testwip/removable-device-completed

(Currently contains a hack to cause Tracker to recognise the fake
mounts as removable devices; we can't create fake Volume or Drive
objects in the test mounter yet due to Vala bug #677769)

7 files changed, 207 insertions, 5 deletions

diff --git a/src/libtracker-miner/tracker-storage.c b/src/libtracker-miner/tracker-storage.c
index 4d0145fda..4acbd592f 100644
--- a/src/libtracker-miner/tracker-storage.c
+++ b/src/libtracker-miner/tracker-storage.c
@@ -682,6 +682,7 @@ mount_add (TrackerStorage *storage,
 		 * Note: Never found a case where this g_mount_get_uuid() returns
 		 * non-NULL... :-) */
 		uuid = g_mount_get_uuid (mount);
+		is_removable = g_mount_can_unmount (mount);
 		if (!uuid) {
 			if (mount_path) {
 				gchar *content_type;
diff --git a/tests/functional-tests/302-miner-removable-devices.py b/tests/functional-tests/302-miner-removable-devices.py
new file mode 100755
index 000000000..89c8058a0
--- /dev/null
+++ b/tests/functional-tests/302-miner-removable-devices.py
@@ -0,0 +1,162 @@
+#!/usr/bin/python
+
+# Copyright (C) 2012, Codethink Ltd. (sam.thursfield@codethink.co.uk)
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA  02110-1301, USA.
+
+"""
+Test miner stability crawling removable devices, which can be disconnected
+and reconnected during mining.
+"""
+
+from common.utils import configuration as cfg
+from common.utils.helpers import MinerFsHelper, StoreHelper, ExtractorHelper, TestMounterHelper, log
+from common.utils.minertest import CommonTrackerMinerTest, MINER_TMP_DIR, get_test_uri, get_test_path
+from common.utils.system import TrackerSystemAbstraction
+
+import os
+import shutil
+import time
+import unittest2 as ut
+
+
+CONF_OPTIONS = [
+    (cfg.DCONF_MINER_SCHEMA, "enable-writeback", "false"),
+    (cfg.DCONF_MINER_SCHEMA, "index-recursive-directories", "[]"),
+    (cfg.DCONF_MINER_SCHEMA, "index-single-directories", "[]"),
+    (cfg.DCONF_MINER_SCHEMA, "index-optical-discs", "true"),
+    (cfg.DCONF_MINER_SCHEMA, "index-removable-devices", "true"),
+    ]
+
+REASONABLE_TIMEOUT = 30
+
+
+class CommonMinerRemovalDevicesTest (CommonTrackerMinerTest):
+    # Use the same instances of store and miner-fs for the whole test suite,
+    # because they take so long to do first-time init.
+    # FIXME: you can move these from all to base class.
+    @classmethod
+    def setUpClass (self):
+        self.system = TrackerSystemAbstraction ()
+        self.system.set_up_environment (CONF_OPTIONS, None)
+
+        self.store = StoreHelper ()
+        self.store.start ()
+        self.miner_fs = MinerFsHelper ()
+        self.miner_fs.start ()
+
+        self.extractor = ExtractorHelper ()
+        self.extractor.start ()
+
+        self.test_mounter_helper = TestMounterHelper ()
+        self.test_mounter_helper.start()
+
+
+    @classmethod
+    def tearDownClass (self):
+        self.test_mounter_helper.stop ();
+        self.extractor.stop ()
+        self.miner_fs.stop ()
+        self.store.stop ()
+
+
+class MinerBasicRemovableDevicesTest (CommonMinerRemovalDevicesTest):
+    @classmethod
+    def setUpClass (self):
+        super(MinerBasicRemovableDevicesTest, self).setUpClass()
+        self._create_test_data_simple ()
+
+    def setUp (self):
+        pass
+
+    def tearDown (self):
+        self.system.unset_up_environment ()
+
+    def test_01_basic (self):
+        """
+        Ensure USB is crawled correctly.
+        """
+        data_dir = os.path.join (MINER_TMP_DIR, 'test-monitored')
+
+        self.test_mounter_helper.mount (data_dir);
+
+        if self.miner_fs.wait_for_device_complete () == False:
+            self.fail ("Timeout waiting for DeviceCompleted notification")
+
+        self.assertEquals (self._get_n_text_documents (), 3)
+        self.test_mounter_helper.unmount (data_dir)
+
+class MinerSlowRemovableDevicesTest (CommonMinerRemovalDevicesTest):
+    @classmethod
+    def setUpClass (self):
+        super(MinerSlowRemovableDevicesTest, self).setUpClass()
+        self._create_test_data_many_files ()
+
+    def setUp (self):
+        pass
+
+    def tearDown (self):
+        self.system.unset_up_environment ()
+
+    def mount_and_remove (self, timeout):
+        self.test_mounter_helper.mount (os.path.join (MINER_TMP_DIR, 'slow-extraction-data'))
+        time.sleep (timeout)
+        self.test_mounter_helper.unmount (os.path.join (MINER_TMP_DIR, 'slow-extraction-data'))
+        self.miner_fs.wait_for_idle (3)
+
+
+    def test_01_quick (self):
+        """
+        Ensure miner functions correctly when USB is unmounted during extraction
+        """
+
+        self.mount_and_remove (2)
+
+        # We shouldn't have finished crawling yet
+        self.miner_fs.wait_for_idle ()
+        self.assertLess (self._get_n_text_documents (), 10000)
+
+        self.assertEqual (self.miner_fs.completed_devices, 0)
+
+    def test_02_in_the_middle (self):
+        """
+        Ensure miner functions correctly when USB is unmounted during extraction
+        """
+
+        self.mount_and_remove (5)
+
+        # We shouldn't have finished crawling yet
+        self.miner_fs.wait_for_idle ()
+        self.assertLess (self._get_n_text_documents (), 10000)
+
+        self.assertEqual (self.miner_fs.completed_devices, 0)
+
+    def test_03_complete (self):
+        """
+        Finally, make sure the miner process hasn't gone crazy
+        """
+
+        self.test_mounter_helper.mount (os.path.join (MINER_TMP_DIR, 'slow-extraction-data'))
+        self.miner_fs.wait_for_device_complete ()
+
+        print "\n\n\n\nGOT DEVICE COMPLETE\n"
+
+        self.assertEquals (self._get_n_text_documents (), 10000)
+
+        self.test_mounter_helper.unmount (os.path.join (MINER_TMP_DIR, 'slow-extraction-data'))
+
+if __name__ == "__main__":
+    ut.main()
diff --git a/tests/functional-tests/Makefile.am b/tests/functional-tests/Makefile.am
index 73e5ef27f..8cd2e10bd 100644
--- a/tests/functional-tests/Makefile.am
+++ b/tests/functional-tests/Makefile.am
@@ -45,7 +45,8 @@ standard_tests += \
 	17-ontology-changes.py  \
 	200-backup-restore.py \
 	300-miner-basic-ops.py \
-	301-miner-resource-removal.py
+	301-miner-resource-removal.py \
+	302-miner-removable-devices.py
 if HAVE_TRACKER_FTS
 standard_tests += 310-fts-indexing.py
 endif
@@ -88,12 +89,12 @@ TEST_RUNNER = $(top_srcdir)/tests/functional-tests/test-runner.sh
 
 functional-test: ${standard_tests}
 	for test in ${standard_tests} ; do \
-		$(TEST_RUNNER) python $(top_srcdir)/tests/functional-tests/$$test; \
+		$(TEST_RUNNER) $(top_srcdir)/tests/functional-tests/$$test; \
 	done
 
 functional-test-slow: ${slow_tests}
 	@for test in ${slow_tests} ; do \
-		$(TEST_RUNNER) python $(top_srcdir)/tests/functional-tests/$$test; \
+		$(TEST_RUNNER) $(top_srcdir)/tests/functional-tests/$$test; \
 	done
 
 EXTRA_DIST = \
diff --git a/tests/functional-tests/common/utils/configuration.py.in b/tests/functional-tests/common/utils/configuration.py.in
index 0ea699d7a..209c677d8 100644
--- a/tests/functional-tests/common/utils/configuration.py.in
+++ b/tests/functional-tests/common/utils/configuration.py.in
@@ -45,6 +45,9 @@ TRACKER_EXTRACT_IFACE = "org.freedesktop.Tracker1.Extract"
 
 WRITEBACK_BUSNAME = "org.freedesktop.Tracker1.Writeback"
 
+TEST_MOUNTER_BUSNAME = "org.freedesktop.Tracker1.TestMounter"
+TEST_MOUNTER_OBJ_PATH = "/org/freedesktop/Tracker1/TestMounter"
+TEST_MOUNTER_IFACE = "org.freedesktop.Tracker1.TestMounter"
 
 DCONF_MINER_SCHEMA = "org.freedesktop.Tracker.Miner.Files"
 
@@ -73,10 +76,12 @@ RAW_EXEC_DIR = "@libexecdir@"
 RAW_DATA_DIR = "@datadir@"
 RAW_DATAROOT_DIR = "@datarootdir@"
 RAW_BINDIR = "@bindir@"
+RAW_LIBDIR = "@libdir@"
 
 EXEC_PREFIX = os.path.normpath (expandvars (RAW_EXEC_DIR))
 DATADIR = os.path.normpath (expandvars (RAW_DATA_DIR))
 BINDIR = os.path.normpath (expandvars (RAW_BINDIR))
+LIBDIR = os.path.normpath (expandvars (RAW_LIBDIR))
                             
 haveMaemo = ("@HAVE_MAEMO_TRUE@" == "")
 haveUpstart = ("@HAVE_UPSTART_TRUE@" == "")
diff --git a/tests/functional-tests/common/utils/helpers.py b/tests/functional-tests/common/utils/helpers.py
index 1301b3faa..a2e1049b8 100644
--- a/tests/functional-tests/common/utils/helpers.py
+++ b/tests/functional-tests/common/utils/helpers.py
@@ -34,7 +34,8 @@ import options
 class NoMetadataException (Exception):
     pass
 
-REASONABLE_TIMEOUT = 30
+#############REASONABLE_TIMEOUT = 30
+REASONABLE_TIMEOUT = 3
 
 def log (message):
     if options.is_verbose ():
@@ -328,6 +329,8 @@ class MinerFsHelper (Helper):
         return False
 
     def _minerfs_status_cb (self, status, progress, remaining_time):
+        log ("Miner status: %s" % status)
+
         if (status == "Idle"):
             self.loop.quit ()
 
@@ -618,3 +621,24 @@ class WritebackHelper (Helper):
     PROCESS_NAME = 'tracker-writeback'
     PROCESS_PATH = os.path.join (cfg.EXEC_PREFIX, 'tracker-writeback')
     BUS_NAME = cfg.WRITEBACK_BUSNAME
+
+
+class TestMounterHelper (Helper):
+
+    PROCESS_NAME = 'tracker-test-mounter-service'
+    PROCESS_PATH = os.path.join (cfg.EXEC_PREFIX, 'tracker-test-mounter-service')
+    BUS_NAME = cfg.TEST_MOUNTER_BUSNAME
+
+    def start (self):
+        Helper.start (self)
+
+        bus_object = self.bus.get_object (cfg.TEST_MOUNTER_BUSNAME,
+                                          cfg.TEST_MOUNTER_OBJ_PATH)
+        self.test_mounter = dbus.Interface (bus_object,
+                                            dbus_interface=cfg.TEST_MOUNTER_IFACE)
+
+    def mount (self, path):
+        self.test_mounter.Mount (path);
+
+    def unmount (self, path):
+        self.test_mounter.Unmount (path)
diff --git a/tests/functional-tests/common/utils/minertest.py b/tests/functional-tests/common/utils/minertest.py
index 7be1bd8b4..054090ed5 100644
--- a/tests/functional-tests/common/utils/minertest.py
+++ b/tests/functional-tests/common/utils/minertest.py
@@ -104,6 +104,15 @@ class CommonTrackerMinerTest (ut.TestCase):
           }
           """)
 
+    def _get_n_text_documents (self):
+        result = self.store.query ("""
+          SELECT COUNT(?u) WHERE {
+              ?u a nfo:TextDocument
+          }
+          """)
+        self.assertEquals (len (result), 1)
+        return int(result[0][0])
+
     def _get_parent_urn (self, filepath):
         result = self.store.query ("""
           SELECT nfo:belongsToContainer(?u) WHERE {
diff --git a/tests/functional-tests/test-runner.sh b/tests/functional-tests/test-runner.sh
index 24c0e4290..5331b41f5 100755
--- a/tests/functional-tests/test-runner.sh
+++ b/tests/functional-tests/test-runner.sh
@@ -31,7 +31,7 @@ else
     trap "/bin/kill $DBUS_SESSION_BUS_PID; exit" INT
 
     echo "Running $@"
-    $@
+    python $@
 
     kill $DBUS_SESSION_BUS_PID
 fi ;