diff options
author | Sam Thursfield <sam@afuera.me.uk> | 2019-08-26 08:20:09 +0000 |
---|---|---|
committer | Sam Thursfield <sam@afuera.me.uk> | 2019-08-26 08:20:09 +0000 |
commit | bdfe556478b5d09a6e5024a413cd1a52e721cc0a (patch) | |
tree | 56ed58126981b275539dc525472a7b76690bf326 | |
parent | b49081beb475334cc36babb15927b0cb3087d210 (diff) | |
parent | 68f73d4302cf48c37d11bbd466befff2f431c228 (diff) | |
download | tracker-bdfe556478b5d09a6e5024a413cd1a52e721cc0a.tar.gz |
Merge branch 'sam/functional-tests-shared' into 'master'
functional-tests: Improve helper code and share it with tracker-miners
See merge request GNOME/tracker!117
40 files changed, 799 insertions, 510 deletions
diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py index 180bca5d3..0bed7aeb3 100755 --- a/tests/functional-tests/01-insertion.py +++ b/tests/functional-tests/01-insertion.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -28,7 +29,7 @@ import random import datetime import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TrackerStoreInsertionTests (CommonTrackerStoreTest): diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py index c5f23674f..4305ea0a9 100755 --- a/tests/functional-tests/02-sparql-bugs.py +++ b/tests/functional-tests/02-sparql-bugs.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -24,7 +25,7 @@ Peculiar Sparql behavour reported in bugs from gi.repository import GLib import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TrackerStoreSparqlBugsTests (CommonTrackerStoreTest): diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py index df2668b22..46c43f368 100755 --- a/tests/functional-tests/03-fts-functions.py +++ b/tests/functional-tests/03-fts-functions.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -23,7 +24,7 @@ These tests use only the store. They insert instances with known text and run sparql with fts functions to check the results. """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TestFTSFunctions (CommonTrackerStoreTest): diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py index 3ae53239a..a8064a828 100755 --- a/tests/functional-tests/04-group-concat.py +++ b/tests/functional-tests/04-group-concat.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,7 +22,7 @@ Test the GROUP_CONCAT function in Sparql. Only requires the store. """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TestGroupConcat (CommonTrackerStoreTest): diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py index a9ef15aab..48d8e6eb6 100755 --- a/tests/functional-tests/05-coalesce.py +++ b/tests/functional-tests/05-coalesce.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,7 +22,7 @@ Test tracker:coalesce function in Sparql. Only uses the Store """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TestCoalesce (CommonTrackerStoreTest): diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py index 52191559f..80d35dfb9 100755 --- a/tests/functional-tests/06-distance.py +++ b/tests/functional-tests/06-distance.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,7 +22,7 @@ Test the distance-calculation functions in Sparql. Only requires the Store """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest POINT_COORDS = [ (0, 0), (1, 1), (2, 2), (3, 3), (4, 4) diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py index 03366fd30..aad935f77 100755 --- a/tests/functional-tests/07-graph.py +++ b/tests/functional-tests/07-graph.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,7 +22,7 @@ Tests graphs in Sparql. Only requires the store. """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TestGraphs (CommonTrackerStoreTest): diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py index 23650231f..9c9578a5c 100755 --- a/tests/functional-tests/08-unique-insertions.py +++ b/tests/functional-tests/08-unique-insertions.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,7 +22,7 @@ Replicate the behaviour of the miner inserting information in the store. """ import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TestMinerInsertBehaviour (CommonTrackerStoreTest): diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py index f362dc98d..b24fdcc40 100755 --- a/tests/functional-tests/09-concurrent-query.py +++ b/tests/functional-tests/09-concurrent-query.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -23,7 +24,7 @@ Send concurrent inserts and queries to the daemon to check the concurrency. from gi.repository import GLib import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest AMOUNT_OF_TEST_INSTANCES = 100 AMOUNT_OF_QUERIES = 10 diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py index 44806e8a2..242ae8480 100755 --- a/tests/functional-tests/14-signals.py +++ b/tests/functional-tests/14-signals.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -23,18 +24,16 @@ are emitted. Theses tests are not extensive (only few selected signals are tested) """ -import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest -from common.utils import configuration as cfg - from gi.repository import Gio from gi.repository import GLib + import time +import unittest as ut + +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest -GRAPH_UPDATED_SIGNAL = "GraphUpdated" -SIGNALS_PATH = "/org/freedesktop/Tracker1/Resources" -SIGNALS_IFACE = "org.freedesktop.Tracker1.Resources" +GRAPH_UPDATED_SIGNAL = "GraphUpdated" CONTACT_CLASS_URI = "http://www.semanticdesktop.org/ontologies/2007/03/22/nco#PersonContact" @@ -70,10 +69,10 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest): After connecting to the signal, call self.__wait_for_signal. """ self.cb_id = self.bus.signal_subscribe( - sender=cfg.TRACKER_BUSNAME, - interface_name=SIGNALS_IFACE, + sender=self.tracker.TRACKER_BUSNAME, + interface_name=self.tracker.RESOURCES_IFACE, member=GRAPH_UPDATED_SIGNAL, - object_path=SIGNALS_PATH, + object_path=self.tracker.TRACKER_OBJ_PATH, arg0=CONTACT_CLASS_URI, flags=Gio.DBusSignalFlags.NONE, callback=self.__signal_received_cb) diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py index ac6a2f210..6f6ca3014 100755 --- a/tests/functional-tests/15-statistics.py +++ b/tests/functional-tests/15-statistics.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,7 +26,7 @@ are updated when different operations are executed on the store import time import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest RDFS_RESOURCE = "rdfs:Resource" NIE_IE = "nie:InformationElement" diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py index 36a1fbbc7..40a993d82 100755 --- a/tests/functional-tests/16-collation.py +++ b/tests/functional-tests/16-collation.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,7 +28,7 @@ import random import locale import unittest as ut -from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest +from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest class TrackerStoreCollationTests (CommonTrackerStoreTest): diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py index 687757fe1..8cf7db220 100755 --- a/tests/functional-tests/17-ontology-changes.py +++ b/tests/functional-tests/17-ontology-changes.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,18 +26,20 @@ changes and checking if the data is still there. from gi.repository import GLib +import logging import os import shutil import re import tempfile import time - -from common.utils import configuration as cfg -from common.utils import helpers -from common.utils.dconf import DConfClient -from common.utils.expectedFailure import expectedFailureJournal import unittest as ut +import trackertestutils.dconf +import trackertestutils.helpers + +import configuration as cfg +from expectedFailure import expectedFailureJournal + RDFS_RANGE = "http://www.w3.org/2000/01/rdf-schema#range" XSD_DATETIME = "http://www.w3.org/2001/XMLSchema#dateTime" @@ -49,6 +52,8 @@ TEST_ENV_VARS = {"LC_COLLATE": "en_GB.utf8"} REASONABLE_TIMEOUT = 5 +log = logging.getLogger() + class UnableToBootException (Exception): pass @@ -88,12 +93,11 @@ class TrackerSystemAbstraction (object): os.environ[var] = directory if ontodir: - helpers.log("export %s=%s" % - ("TRACKER_DB_ONTOLOGIES_DIR", ontodir)) + log.debug("export %s=%s", "TRACKER_DB_ONTOLOGIES_DIR", ontodir) os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir for var, value in TEST_ENV_VARS.items(): - helpers.log("export %s=%s" % (var, value)) + log.debug("export %s=%s", var, value) os.environ[var] = value # Previous loop should have set DCONF_PROFILE to the test location @@ -102,7 +106,7 @@ class TrackerSystemAbstraction (object): def _apply_settings(self, settings): for schema_name, contents in settings.items(): - dconf = DConfClient(schema_name) + dconf = trackertestutils.dconf.DConfClient(schema_name) dconf.reset() for key, value in contents.items(): dconf.write(key, value) @@ -114,7 +118,7 @@ class TrackerSystemAbstraction (object): """ self.set_up_environment(confdir, ontodir) - self.store = helpers.StoreHelper() + self.store = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH) self.store.start() def tracker_store_restart_with_new_ontologies(self, ontodir): diff --git a/tests/functional-tests/__init__.py b/tests/functional-tests/__init__.py index a93a4bf16..e69de29bb 100644 --- a/tests/functional-tests/__init__.py +++ b/tests/functional-tests/__init__.py @@ -1 +0,0 @@ -#!/usr/bin/python3 diff --git a/tests/functional-tests/common/__init__.py b/tests/functional-tests/common/__init__.py deleted file mode 100644 index a93a4bf16..000000000 --- a/tests/functional-tests/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/python3 diff --git a/tests/functional-tests/common/data/Doc/performance.doc b/tests/functional-tests/common/data/Doc/performance.doc Binary files differdeleted file mode 100644 index 8450cdd63..000000000 --- a/tests/functional-tests/common/data/Doc/performance.doc +++ /dev/null diff --git a/tests/functional-tests/common/data/Images/test-image-2.png b/tests/functional-tests/common/data/Images/test-image-2.png Binary files differdeleted file mode 100644 index 7ff9788a0..000000000 --- a/tests/functional-tests/common/data/Images/test-image-2.png +++ /dev/null diff --git a/tests/functional-tests/common/data/Images/test-image-3.tif b/tests/functional-tests/common/data/Images/test-image-3.tif Binary files differdeleted file mode 100644 index 8d91556a7..000000000 --- a/tests/functional-tests/common/data/Images/test-image-3.tif +++ /dev/null diff --git a/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf b/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf Binary files differdeleted file mode 100644 index 064645c39..000000000 --- a/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf +++ /dev/null diff --git a/tests/functional-tests/common/data/Ppt/al-cont.ppt b/tests/functional-tests/common/data/Ppt/al-cont.ppt Binary files differdeleted file mode 100644 index 2fc0a733f..000000000 --- a/tests/functional-tests/common/data/Ppt/al-cont.ppt +++ /dev/null diff --git a/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg b/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg Binary files differdeleted file mode 100644 index f1f917bb5..000000000 --- a/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg +++ /dev/null diff --git a/tests/functional-tests/common/data/pickled_Images b/tests/functional-tests/common/data/pickled_Images Binary files differdeleted file mode 100644 index 330cf46c2..000000000 --- a/tests/functional-tests/common/data/pickled_Images +++ /dev/null diff --git a/tests/functional-tests/common/data/pickled_Music b/tests/functional-tests/common/data/pickled_Music Binary files differdeleted file mode 100644 index 1913fc4bf..000000000 --- a/tests/functional-tests/common/data/pickled_Music +++ /dev/null diff --git a/tests/functional-tests/common/utils/__init__.py b/tests/functional-tests/common/utils/__init__.py deleted file mode 100644 index a93a4bf16..000000000 --- a/tests/functional-tests/common/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/python3 diff --git a/tests/functional-tests/common/utils/configuration.py b/tests/functional-tests/common/utils/configuration.py deleted file mode 100644 index 1a327ab88..000000000 --- a/tests/functional-tests/common/utils/configuration.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com> -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -# 02110-1301, USA. -# - -"Constants describing Tracker D-Bus services" - -import json -import os - -if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ: - raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment " - "variable must be set to point to the location of " - "the generated configuration.json file.") - -with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f: - config = json.load(f) - -TRACKER_BUSNAME = 'org.freedesktop.Tracker1' -TRACKER_OBJ_PATH = '/org/freedesktop/Tracker1/Resources' -RESOURCES_IFACE = "org.freedesktop.Tracker1.Resources" - -MINERFS_BUSNAME = "org.freedesktop.Tracker1.Miner.Files" -MINERFS_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Files" -MINER_IFACE = "org.freedesktop.Tracker1.Miner" -MINERFS_INDEX_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Files/Index" -MINER_INDEX_IFACE = "org.freedesktop.Tracker1.Miner.Files.Index" - -TRACKER_BACKUP_OBJ_PATH = "/org/freedesktop/Tracker1/Backup" -BACKUP_IFACE = "org.freedesktop.Tracker1.Backup" - -TRACKER_STATS_OBJ_PATH = "/org/freedesktop/Tracker1/Statistics" -STATS_IFACE = "org.freedesktop.Tracker1.Statistics" - -TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status" -STATUS_IFACE = "org.freedesktop.Tracker1.Status" - -TRACKER_EXTRACT_BUSNAME = "org.freedesktop.Tracker1.Miner.Extract" -TRACKER_EXTRACT_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Extract" - -WRITEBACK_BUSNAME = "org.freedesktop.Tracker1.Writeback" - - -DCONF_MINER_SCHEMA = "org.freedesktop.Tracker.Miner.Files" - -# Autoconf substitutes paths in the configuration.json file without -# expanding variables, so we need to manually insert these. - - -def expandvars(variable): - # Note: the order matters! - result = variable - for var, value in [("${datarootdir}", RAW_DATAROOT_DIR), - ("${exec_prefix}", RAW_EXEC_PREFIX), - ("${prefix}", PREFIX), - ("@top_srcdir@", TOP_SRCDIR), - ("@top_builddir@", TOP_BUILDDIR)]: - result = result.replace(var, value) - - return result - - -PREFIX = config['PREFIX'] -RAW_EXEC_PREFIX = config['RAW_EXEC_PREFIX'] -RAW_DATAROOT_DIR = config['RAW_DATAROOT_DIR'] - -TOP_SRCDIR = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) -TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR'] - -TEST_ONTOLOGIES_DIR = os.path.normpath( - expandvars(config['TEST_ONTOLOGIES_DIR'])) - -TRACKER_STORE_PATH = os.path.normpath(expandvars(config['TRACKER_STORE_PATH'])) - -disableJournal = (len(config['disableJournal']) == 0) diff --git a/tests/functional-tests/common/utils/helpers.py b/tests/functional-tests/common/utils/helpers.py deleted file mode 100644 index 6e5c26c40..000000000 --- a/tests/functional-tests/common/utils/helpers.py +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com> -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -# 02110-1301, USA. -# -from gi.repository import Gio -from gi.repository import GLib -import os -import sys -import subprocess -import time -import re - -from common.utils import configuration as cfg -from common.utils import options as options - - -class NoMetadataException (Exception): - pass - -REASONABLE_TIMEOUT = 30 - - -def log(message): - if options.is_verbose(): - print(message) - - -class Helper: - """ - Abstract helper for Tracker processes. Launches the process manually - and waits for it to appear on the session bus. - - The helper will fail if the process is already running. Use - test-runner.sh to ensure the processes run inside a separate DBus - session bus. - - The process is watched using a timed GLib main loop source. If the process - exits with an error code, the test will abort the next time the main loop - is entered (or straight away if currently running the main loop). - """ - - BUS_NAME = None - PROCESS_NAME = None - - def __init__(self): - self.process = None - self.available = False - - self.loop = GLib.MainLoop() - self.install_glib_excepthook(self.loop) - - self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) - - def install_glib_excepthook(self, loop): - """ - Handler to abort test if an exception occurs inside the GLib main loop. - """ - old_hook = sys.excepthook - - def new_hook(etype, evalue, etb): - old_hook(etype, evalue, etb) - GLib.MainLoop.quit(loop) - sys.exit(1) - sys.excepthook = new_hook - - def _start_process(self, env=None): - path = self.PROCESS_PATH - flags = getattr(self, - "FLAGS", - []) - - kws = {} - - if not options.is_verbose(): - FNULL = open('/dev/null', 'w') - kws.update({'stdout': FNULL, 'stderr': subprocess.PIPE}) - - if env: - kws['env'] = env - - command = [path] + flags - log("Starting %s" % ' '.join(command)) - try: - return subprocess.Popen([path] + flags, **kws) - except OSError as e: - raise RuntimeError("Error starting %s: %s" % (path, e)) - - def _bus_name_appeared(self, name, owner, data): - log("[%s] appeared in the bus as %s" % (self.PROCESS_NAME, owner)) - self.available = True - self.loop.quit() - - def _bus_name_vanished(self, name, data): - log("[%s] disappeared from the bus" % self.PROCESS_NAME) - self.available = False - self.loop.quit() - - def _process_watch_cb(self): - if self.process_watch_timeout == 0: - # The GLib seems to call the timeout after we've removed it - # sometimes, which causes errors unless we detect it. - return False - - status = self.process.poll() - - if status is None: - return True # continue - elif status == 0 and not self.abort_if_process_exits_with_status_0: - return True # continue - else: - self.process_watch_timeout = 0 - if options.is_verbose(): - error = "" - else: - error = self.process.stderr.read() - raise RuntimeError("%s exited with status: %i\n%s" % - (self.PROCESS_NAME, status, error)) - - def _timeout_on_idle_cb(self): - log("[%s] Timeout waiting... asumming idle." % self.PROCESS_NAME) - self.loop.quit() - self.timeout_id = None - return False - - def start(self, env=None): - """ - Start an instance of process and wait for it to appear on the bus. - """ - if self.process is not None: - raise RuntimeError( - "%s process already started" % self.PROCESS_NAME) - - self._bus_name_watch_id = Gio.bus_watch_name_on_connection( - self.bus, self.BUS_NAME, Gio.BusNameWatcherFlags.NONE, - self._bus_name_appeared, self._bus_name_vanished) - self.loop.run() - - if options.is_manual_start(): - print("Start %s manually" % self.PROCESS_NAME) - else: - if self.available: - # It's running, but we didn't start it... - raise Exception("Unable to start test instance of %s: " - "already running " % self.PROCESS_NAME) - - self.process = self._start_process(env=env) - log('[%s] Started process %i' % - (self.PROCESS_NAME, self.process.pid)) - self.process_watch_timeout = GLib.timeout_add( - 200, self._process_watch_cb) - - self.abort_if_process_exits_with_status_0 = True - - # Run the loop until the bus name appears, or the process dies. - self.loop.run() - - self.abort_if_process_exits_with_status_0 = False - - def stop(self): - if self.process is None: - # Seems that it didn't even start... - return - - start = time.time() - if self.process.poll() == None: - GLib.source_remove(self.process_watch_timeout) - self.process_watch_timeout = 0 - - self.process.terminate() - - while self.process.poll() == None: - time.sleep(0.1) - - if time.time() > (start + REASONABLE_TIMEOUT): - log("[%s] Failed to terminate, sending kill!" % - self.PROCESS_NAME) - self.process.kill() - self.process.wait() - - log("[%s] stopped." % self.PROCESS_NAME) - - # Run the loop until the bus name appears, or the process dies. - self.loop.run() - Gio.bus_unwatch_name(self._bus_name_watch_id) - - self.process = None - - def kill(self): - if options.is_manual_start(): - log("kill(): ignoring, because process was started manually.") - return - - if self.process_watch_timeout != 0: - GLib.source_remove(self.process_watch_timeout) - self.process_watch_timeout = 0 - - self.process.kill() - - # Name owner changed callback should take us out from this loop - self.loop.run() - Gio.bus_unwatch_name(self._bus_name_watch_id) - - self.process = None - - log("[%s] killed." % self.PROCESS_NAME) - - -class StoreHelper (Helper): - """ - Wrapper for the Store API - - Every method tries to reconnect once if there is a dbus exception - (some tests kill the daemon and make the connection useless) - """ - - PROCESS_NAME = "tracker-store" - PROCESS_PATH = cfg.TRACKER_STORE_PATH - BUS_NAME = cfg.TRACKER_BUSNAME - - def start(self, env=None): - Helper.start(self, env=env) - - self.resources = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, - cfg.TRACKER_BUSNAME, cfg.TRACKER_OBJ_PATH, cfg.RESOURCES_IFACE) - - self.backup_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, - cfg.TRACKER_BUSNAME, cfg.TRACKER_BACKUP_OBJ_PATH, cfg.BACKUP_IFACE) - - self.stats_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, - cfg.TRACKER_BUSNAME, cfg.TRACKER_STATS_OBJ_PATH, cfg.STATS_IFACE) - - self.status_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, - cfg.TRACKER_BUSNAME, cfg.TRACKER_STATUS_OBJ_PATH, cfg.STATUS_IFACE) - - log("[%s] booting..." % self.PROCESS_NAME) - self.status_iface.Wait() - log("[%s] ready." % self.PROCESS_NAME) - - def stop(self): - Helper.stop(self) - - def query(self, query, timeout=5000, **kwargs): - return self.resources.SparqlQuery('(s)', query, timeout=timeout, **kwargs) - - def update(self, update_sparql, timeout=5000, **kwargs): - return self.resources.SparqlUpdate('(s)', update_sparql, timeout=timeout, **kwargs) - - def load(self, ttl_uri, timeout=5000, **kwargs): - return self.resources.Load('(s)', ttl_uri, timeout=timeout, **kwargs) - - def batch_update(self, update_sparql, **kwargs): - return self.resources.BatchSparqlUpdate('(s)', update_sparql, **kwargs) - - def batch_commit(self, **kwargs): - return self.resources.BatchCommit(**kwargs) - - def backup(self, backup_file, **kwargs): - return self.backup_iface.Save('(s)', backup_file, **kwargs) - - def restore(self, backup_file, **kwargs): - return self.backup_iface.Restore('(s)', backup_file, **kwargs) - - def get_stats(self, **kwargs): - return self.stats_iface.Get(**kwargs) - - def get_tracker_iface(self): - return self.resources - - def count_instances(self, ontology_class): - QUERY = """ - SELECT COUNT(?u) WHERE { - ?u a %s . - } - """ - result = self.resources.SparqlQuery('(s)', QUERY % (ontology_class)) - - if (len(result) == 1): - return int(result[0][0]) - else: - return -1 - - def get_resource_id_by_uri(self, uri): - """ - Get the internal ID for a given resource, identified by URI. - """ - result = self.query( - 'SELECT tracker:id(%s) WHERE { }' % uri) - if len(result) == 1: - return int(result[0][0]) - elif len(result) == 0: - raise Exception("No entry for resource %s" % uri) - else: - raise Exception("Multiple entries for resource %s" % uri) - - # FIXME: rename to get_resource_id_by_nepomuk_url !! - def get_resource_id(self, url): - """ - Get the internal ID for a given resource, identified by URL. - """ - result = self.query( - 'SELECT tracker:id(?r) WHERE { ?r nie:url "%s" }' % url) - if len(result) == 1: - return int(result[0][0]) - elif len(result) == 0: - raise Exception("No entry for resource %s" % url) - else: - raise Exception("Multiple entries for resource %s" % url) - - def ask(self, ask_query): - assert ask_query.strip().startswith("ASK") - result = self.query(ask_query) - assert len(result) == 1 - if result[0][0] == "true": - return True - elif result[0][0] == "false": - return False - else: - raise Exception("Something fishy is going on") diff --git a/tests/functional-tests/common/utils/options.py b/tests/functional-tests/common/utils/options.py deleted file mode 100644 index d11d5a12f..000000000 --- a/tests/functional-tests/common/utils/options.py +++ /dev/null @@ -1,28 +0,0 @@ -import os - - -def get_environment_boolean(variable): - '''Parse a yes/no boolean passed through the environment.''' - - value = os.environ.get(variable, 'no').lower() - if value in ['no', '0', 'false']: - return False - elif value in ['yes', '1', 'true']: - return True - else: - raise RuntimeError('Unexpected value for %s: %s' % - (variable, value)) - - -def is_verbose(): - """ - True to log process status information to stdout - """ - return get_environment_boolean('TRACKER_TESTS_VERBOSE') - - -def is_manual_start(): - """ - False to start the processes automatically - """ - return get_environment_boolean('TRACKER_TESTS_MANUAL_START') diff --git a/tests/functional-tests/common/utils/system.py b/tests/functional-tests/common/utils/system.py deleted file mode 100644 index a93a4bf16..000000000 --- a/tests/functional-tests/common/utils/system.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/python3 diff --git a/tests/functional-tests/configuration.json.in b/tests/functional-tests/configuration.json.in index 06a5be195..46c5126f7 100644 --- a/tests/functional-tests/configuration.json.in +++ b/tests/functional-tests/configuration.json.in @@ -1,7 +1,4 @@ { - "PREFIX": "@prefix@", - "RAW_EXEC_PREFIX": "@exec_prefix@", - "RAW_DATAROOT_DIR": "@datarootdir@", "TEST_ONTOLOGIES_DIR": "@FUNCTIONAL_TESTS_ONTOLOGIES_DIR@", "TRACKER_STORE_PATH": "@FUNCTIONAL_TESTS_TRACKER_STORE_PATH@", "disableJournal": "@DISABLE_JOURNAL_TRUE@" diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py new file mode 100644 index 000000000..938ad0f19 --- /dev/null +++ b/tests/functional-tests/configuration.py @@ -0,0 +1,60 @@ +# +# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + + +import json +import logging +import os +import sys + + +if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ: + raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment " + "variable must be set to point to the location of " + "the generated configuration.json file.") + +with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f: + config = json.load(f) + + +TOP_SRCDIR = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR'] + +TEST_ONTOLOGIES_DIR = config['TEST_ONTOLOGIES_DIR'] +TRACKER_STORE_PATH = config['TRACKER_STORE_PATH'] +disableJournal = (len(config['disableJournal']) == 0) + + +def get_environment_boolean(variable): + '''Parse a yes/no boolean passed through the environment.''' + + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: + return False + elif value in ['yes', '1', 'true']: + return True + else: + raise RuntimeError('Unexpected value for %s: %s' % + (variable, value)) + + +if get_environment_boolean('TRACKER_TESTS_VERBOSE'): + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) diff --git a/tests/functional-tests/common/utils/expectedFailure.py b/tests/functional-tests/expectedFailure.py index e7f0eb68c..289e0a9df 100644 --- a/tests/functional-tests/common/utils/expectedFailure.py +++ b/tests/functional-tests/expectedFailure.py @@ -1,10 +1,9 @@ -#!/usr/bin/python3 - # Code taken and modified from unittest2 framework (case.py) # Copyright (c) 1999-2003 Steve Purcell # Copyright (c) 2003-2010 Python Software Foundation # Copyright (c) 2010, Nokia (ivan.frade@nokia.com) +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> # This module is free software, and you may redistribute it and/or modify # it under the same terms as Python itself, so long as this copyright message @@ -25,9 +24,11 @@ Write values in tracker and check the actual values are written on the files. Note that these tests are highly platform dependant. """ -import sys + from functools import wraps -import common.utils.configuration as cfg +import sys + +import configuration as cfg def expectedFailureJournal(): diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build index c482fc3df..be3fc2a4a 100644 --- a/tests/functional-tests/meson.build +++ b/tests/functional-tests/meson.build @@ -33,6 +33,9 @@ test_env = environment() test_env.set('DCONF_PROFILE', dconf_profile_full_path) test_env.set('GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir) +tracker_uninstalled_testutils_dir = join_paths(meson.current_source_dir(), '..', '..', 'utils') +test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir) + test_env.set('TRACKER_DB_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir) test_env.set('TRACKER_FUNCTIONAL_TEST_BUILD_DIR', build_root) test_env.set('TRACKER_FUNCTIONAL_TEST_CONFIG', config_json_full_path) diff --git a/tests/functional-tests/common/utils/storetest.py b/tests/functional-tests/storetest.py index dad9a0ac6..ed7aa82c5 100644 --- a/tests/functional-tests/common/utils/storetest.py +++ b/tests/functional-tests/storetest.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python # # Copyright (C) 2010, Nokia <ivan.frade@nokia.com> -# Copyright (C) 2018, Sam Thursfield <sam@afuera.me.uk> +# Copyright (C) 2018, 2019, Sam Thursfield <sam@afuera.me.uk> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -19,26 +18,27 @@ # 02110-1301, USA. # -import unittest as ut - import os import time +import unittest as ut -from common.utils.helpers import StoreHelper -from common.utils import configuration as cfg +import trackertestutils.helpers +import configuration as cfg class CommonTrackerStoreTest (ut.TestCase): """ Common superclass for tests that just require a fresh store running + + Note that the store is started per test-suite, not per test. """ + @classmethod def setUpClass(self): - env = os.environ - env['LC_COLLATE'] = 'en_GB.utf8' + extra_env = {'LC_COLLATE': 'en_GB.utf8'} - self.tracker = StoreHelper() - self.tracker.start(env=env) + self.tracker = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH) + self.tracker.start(extra_env=extra_env) @classmethod def tearDownClass(self): diff --git a/utils/meson.build b/utils/meson.build index c624b4914..3b7847501 100644 --- a/utils/meson.build +++ b/utils/meson.build @@ -1,3 +1,4 @@ subdir('mtp') subdir('ontology') subdir('tracker-resdump') +subdir('trackertestutils') diff --git a/utils/trackertestutils/README.md b/utils/trackertestutils/README.md new file mode 100644 index 000000000..98840ba0a --- /dev/null +++ b/utils/trackertestutils/README.md @@ -0,0 +1,2 @@ +This Python package contains utility functions which are useful when testing +Tracker. diff --git a/utils/trackertestutils/__init__.py b/utils/trackertestutils/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/utils/trackertestutils/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/functional-tests/common/utils/dconf.py b/utils/trackertestutils/dconf.py index d4b42dfe9..4ad0e88e9 100644 --- a/tests/functional-tests/common/utils/dconf.py +++ b/utils/trackertestutils/dconf.py @@ -1,9 +1,30 @@ +# +# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + from gi.repository import GLib from gi.repository import Gio +import logging import os -from common.utils.helpers import log +log = logging.getLogger(__name__) class DConfClient(object): @@ -75,5 +96,5 @@ class DConfClient(object): "dconf", "trackertest") if os.path.exists(dconf_db): - log("[Conf] Removing dconf database: " + dconf_db) + log.debug("[Conf] Removing dconf database: %s", dconf_db) os.remove(dconf_db) diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py new file mode 100644 index 000000000..2b218e5d0 --- /dev/null +++ b/utils/trackertestutils/helpers.py @@ -0,0 +1,583 @@ +# +# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com> +# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +from gi.repository import Gio +from gi.repository import GLib + +import atexit +import logging +import os +import subprocess + +from . import mainloop + +log = logging.getLogger(__name__) + + +class GraphUpdateTimeoutException(RuntimeError): + pass + + +class NoMetadataException (Exception): + pass + + +REASONABLE_TIMEOUT = 30 + + +_process_list = [] + + +def _cleanup_processes(): + for process in _process_list: + log.debug("helpers._cleanup_processes: stopping %s", process) + process.stop() + + +atexit.register(_cleanup_processes) + + +class Helper: + """ + Abstract helper for Tracker processes. Launches the process + and waits for it to appear on the session bus. + + The helper will fail if the process is already running. Use + test-runner.sh to ensure the processes run inside a separate DBus + session bus. + + The process is watched using a timed GLib main loop source. If the process + exits with an error code, the test will abort the next time the main loop + is entered (or straight away if currently running the main loop). + """ + + STARTUP_TIMEOUT = 200 # milliseconds + SHUTDOWN_TIMEOUT = 200 # + + def __init__(self, helper_name, bus_name, process_path): + self.name = helper_name + self.bus_name = bus_name + self.process_path = process_path + + self.log = logging.getLogger(f'{__name__}.{self.name}') + + self.process = None + self.available = False + + self.loop = mainloop.MainLoop() + + self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) + + def _start_process(self, command_args=None, extra_env=None): + global _process_list + _process_list.append(self) + + command = [self.process_path] + (command_args or []) + self.log.debug("Starting %s.", ' '.join(command)) + + env = os.environ + if extra_env: + self.log.debug(" starting with extra environment: %s", extra_env) + env.update(extra_env) + + try: + return subprocess.Popen(command, env=env) + except OSError as e: + raise RuntimeError("Error starting %s: %s" % (self.process_path, e)) + + def _bus_name_appeared(self, connection, name, owner): + self.log.debug("%s appeared on the message bus, owned by %s", name, owner) + self.available = True + self.loop.quit() + + def _bus_name_vanished(self, connection, name): + self.log.debug("%s vanished from the message bus", name) + self.available = False + self.loop.quit() + + def _process_watch_cb(self): + if self.process_watch_timeout == 0: + # GLib seems to call the timeout after we've removed it + # sometimes, which causes errors unless we detect it. + return False + + status = self.process.poll() + + if status is None: + return True # continue + elif status == 0 and not self.abort_if_process_exits_with_status_0: + return True # continue + else: + self.process_watch_timeout = 0 + raise RuntimeError(f"{self.name} exited with status: {self.status}") + + def _process_startup_timeout_cb(self): + self.log.debug(f"Process timeout of {self.STARTUP_TIMEOUT}ms was called") + self.loop.quit() + self.timeout_id = None + return False + + def start(self, command_args=None, extra_env=None): + """ + Start an instance of process and wait for it to appear on the bus. + """ + if self.process is not None: + raise RuntimeError("%s: already started" % self.name) + + self._bus_name_watch_id = Gio.bus_watch_name_on_connection( + self.bus, self.bus_name, Gio.BusNameWatcherFlags.NONE, + self._bus_name_appeared, self._bus_name_vanished) + + # We expect the _bus_name_vanished callback to be called here, + # causing the loop to exit again. + self.loop.run_checked() + + if self.available: + # It's running, but we didn't start it... + raise RuntimeError("Unable to start test instance of %s: " + "already running" % self.name) + + self.process = self._start_process(command_args=command_args, + extra_env=extra_env) + self.log.debug('Started with PID %i', self.process.pid) + + self.process_startup_timeout = GLib.timeout_add( + self.STARTUP_TIMEOUT, self._process_startup_timeout_cb) + + self.abort_if_process_exits_with_status_0 = True + + # Run the loop until the bus name appears, or the process dies. + self.loop.run_checked() + + self.abort_if_process_exits_with_status_0 = False + + def stop(self): + global _process_list + + if self.process is None: + # Seems that it didn't even start... + return + + if self.process.poll() == None: + GLib.source_remove(self.process_startup_timeout) + self.process_startup_timeout = 0 + + self.process.terminate() + returncode = self.process.wait(timeout=self.SHUTDOWN_TIMEOUT * 1000) + if returncode is None: + self.log.debug("Process failed to terminate in time, sending kill!") + self.process.kill() + self.process.wait() + elif returncode > 0: + self.log.warn("Process returned error code %s", returncode) + + self.log.debug("Process stopped.") + + # Run the loop to handle the expected name_vanished signal. + self.loop.run_checked() + Gio.bus_unwatch_name(self._bus_name_watch_id) + + self.process = None + _process_list.remove(self) + + def kill(self): + global _process_list + + if self.process_watch_timeout != 0: + GLib.source_remove(self.process_watch_timeout) + self.process_watch_timeout = 0 + + self.process.kill() + + # Name owner changed callback should take us out from this loop + self.loop.run_checked() + Gio.bus_unwatch_name(self._bus_name_watch_id) + + self.process = None + _process_list.remove(self) + + self.log.debug("Process killed.") + + +class StoreHelper (Helper): + """ + Helper for starting and testing the tracker-store daemon. + """ + + TRACKER_BUSNAME = 'org.freedesktop.Tracker1' + TRACKER_OBJ_PATH = '/org/freedesktop/Tracker1/Resources' + RESOURCES_IFACE = "org.freedesktop.Tracker1.Resources" + + TRACKER_BACKUP_OBJ_PATH = "/org/freedesktop/Tracker1/Backup" + BACKUP_IFACE = "org.freedesktop.Tracker1.Backup" + + TRACKER_STATS_OBJ_PATH = "/org/freedesktop/Tracker1/Statistics" + STATS_IFACE = "org.freedesktop.Tracker1.Statistics" + + TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status" + STATUS_IFACE = "org.freedesktop.Tracker1.Status" + + def __init__(self, process_path): + Helper.__init__(self, "tracker-store", self.TRACKER_BUSNAME, process_path) + + def start(self, command_args=None, extra_env=None): + Helper.start(self, command_args, extra_env) + + self.resources = Gio.DBusProxy.new_sync( + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.TRACKER_BUSNAME, self.TRACKER_OBJ_PATH, self.RESOURCES_IFACE) + + self.backup_iface = Gio.DBusProxy.new_sync( + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.TRACKER_BUSNAME, self.TRACKER_BACKUP_OBJ_PATH, self.BACKUP_IFACE) + + self.stats_iface = Gio.DBusProxy.new_sync( + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.TRACKER_BUSNAME, self.TRACKER_STATS_OBJ_PATH, self.STATS_IFACE) + + self.status_iface = Gio.DBusProxy.new_sync( + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.TRACKER_BUSNAME, self.TRACKER_STATUS_OBJ_PATH, self.STATUS_IFACE) + + self.log.debug("Calling %s.Wait() method", self.STATUS_IFACE) + self.status_iface.Wait() + self.log.debug("Ready") + + self.reset_graph_updates_tracking() + + def signal_handler(proxy, sender_name, signal_name, parameters): + if signal_name == 'GraphUpdated': + self._graph_updated_cb(*parameters.unpack()) + + self.graph_updated_handler_id = self.resources.connect( + 'g-signal', signal_handler) + + def stop(self): + Helper.stop(self) + + if self.graph_updated_handler_id != 0: + self.resources.disconnect(self.graph_updated_handler_id) + + # A system to follow GraphUpdated and make sure all changes are tracked. + # This code saves every change notification received, and exposes methods + # to await insertion or deletion of a certain resource which first check + # the list of events already received and wait for more if the event has + # not yet happened. + + def reset_graph_updates_tracking(self): + self.class_to_track = None + self.inserts_list = [] + self.deletes_list = [] + self.inserts_match_function = None + self.deletes_match_function = None + + def _graph_updated_timeout_cb(self): + raise GraphUpdateTimeoutException() + + def _graph_updated_cb(self, class_name, deletes_list, inserts_list): + """ + Process notifications from tracker-store on resource changes. + """ + exit_loop = False + + if class_name == self.class_to_track: + self.log.debug("GraphUpdated for %s: %i deletes, %i inserts", class_name, len(deletes_list), len(inserts_list)) + + if inserts_list is not None: + if self.inserts_match_function is not None: + # The match function will remove matched entries from the list + (exit_loop, inserts_list) = self.inserts_match_function(inserts_list) + self.inserts_list += inserts_list + + if not exit_loop and deletes_list is not None: + if self.deletes_match_function is not None: + (exit_loop, deletes_list) = self.deletes_match_function(deletes_list) + self.deletes_list += deletes_list + + if exit_loop: + GLib.source_remove(self.graph_updated_timeout_id) + self.graph_updated_timeout_id = 0 + self.loop.quit() + else: + self.log.debug("Ignoring GraphUpdated for class %s, currently tracking %s", class_name, self.class_to_track) + + def _enable_await_timeout(self): + self.graph_updated_timeout_id = GLib.timeout_add_seconds(REASONABLE_TIMEOUT, + self._graph_updated_timeout_cb) + + def await_resource_inserted(self, rdf_class, url=None, title=None, required_property=None): + """ + Block until a resource matching the parameters becomes available + """ + assert (self.inserts_match_function == None) + assert (self.class_to_track == None), "Already waiting for resource of type %s" % self.class_to_track + + self.class_to_track = rdf_class + + self.matched_resource_urn = None + self.matched_resource_id = None + + self.log.debug("Await new %s (%i existing inserts)", rdf_class, len(self.inserts_list)) + + if required_property is not None: + required_property_id = self.get_resource_id_by_uri(required_property) + self.log.debug("Required property %s id %i", required_property, required_property_id) + + def find_resource_insertion(inserts_list): + matched_creation = (self.matched_resource_id is not None) + matched_required_property = False + remaining_events = [] + + # FIXME: this could be done in an easier way: build one query that filters + # based on every subject id in inserts_list, and returns the id of the one + # that matched :) + for insert in inserts_list: + id = insert[1] + + if not matched_creation: + where = " ?urn a <%s> " % rdf_class + + if url is not None: + where += "; nie:url \"%s\"" % url + + if title is not None: + where += "; nie:title \"%s\"" % title + + query = "SELECT ?urn WHERE { %s FILTER (tracker:id(?urn) = %s)}" % (where, insert[1]) + result_set = self.query(query) + + if len(result_set) > 0: + matched_creation = True + self.matched_resource_urn = result_set[0][0] + self.matched_resource_id = insert[1] + self.log.debug("Matched creation of resource %s (%i)", + self.matched_resource_urn, + self.matched_resource_id) + if required_property is not None: + self.log.debug("Waiting for property %s (%i) to be set", + required_property, required_property_id) + + if required_property is not None and matched_creation and not matched_required_property: + if id == self.matched_resource_id and insert[2] == required_property_id: + matched_required_property = True + self.log.debug("Matched %s %s", self.matched_resource_urn, required_property) + + if not matched_creation or id != self.matched_resource_id: + remaining_events += [insert] + + matched = matched_creation if required_property is None else matched_required_property + return matched, remaining_events + + def match_cb(inserts_list): + matched, remaining_events = find_resource_insertion(inserts_list) + exit_loop = matched + return exit_loop, remaining_events + + # Check the list of previously received events for matches + (existing_match, self.inserts_list) = find_resource_insertion(self.inserts_list) + + if not existing_match: + self._enable_await_timeout() + self.inserts_match_function = match_cb + # Run the event loop until the correct notification arrives + try: + self.loop.run_checked() + except GraphUpdateTimeoutException: + raise GraphUpdateTimeoutException("Timeout waiting for resource: class %s, URL %s, title %s" % (rdf_class, url, title)) from None + self.inserts_match_function = None + + self.class_to_track = None + return (self.matched_resource_id, self.matched_resource_urn) + + def await_resource_deleted(self, rdf_class, id): + """ + Block until we are notified of a resources deletion + """ + assert (self.deletes_match_function == None) + assert (self.class_to_track == None) + + def find_resource_deletion(deletes_list): + self.log.debug("find_resource_deletion: looking for %i in %s", id, deletes_list) + + matched = False + remaining_events = [] + + for delete in deletes_list: + if delete[1] == id: + matched = True + else: + remaining_events += [delete] + + return matched, remaining_events + + def match_cb(deletes_list): + matched, remaining_events = find_resource_deletion(deletes_list) + exit_loop = matched + return exit_loop, remaining_events + + self.log.debug("Await deletion of %i (%i existing)", id, len(self.deletes_list)) + + (existing_match, self.deletes_list) = find_resource_deletion(self.deletes_list) + + if not existing_match: + self._enable_await_timeout() + self.class_to_track = rdf_class + self.deletes_match_function = match_cb + # Run the event loop until the correct notification arrives + try: + self.loop.run_checked() + except GraphUpdateTimeoutException: + raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id) + self.deletes_match_function = None + self.class_to_track = None + + return + + def await_property_changed(self, rdf_class, subject_id, property_uri): + """ + Block until a property of a resource is updated or inserted. + """ + assert (self.inserts_match_function == None) + assert (self.deletes_match_function == None) + assert (self.class_to_track == None) + + self.log.debug("Await change to %i %s (%i, %i existing)", subject_id, property_uri, len(self.inserts_list), len(self.deletes_list)) + + self.class_to_track = rdf_class + + property_id = self.get_resource_id_by_uri(property_uri) + + def find_property_change(event_list): + matched = False + remaining_events = [] + + for event in event_list: + if event[1] == subject_id and event[2] == property_id: + self.log.debug("Matched property change: %s", str(event)) + matched = True + else: + remaining_events += [event] + + return matched, remaining_events + + def match_cb(event_list): + matched, remaining_events = find_property_change(event_list) + exit_loop = matched + return exit_loop, remaining_events + + # Check the list of previously received events for matches + (existing_match, self.inserts_list) = find_property_change(self.inserts_list) + (existing_match, self.deletes_list) = find_property_change(self.deletes_list) + + if not existing_match: + self._enable_await_timeout() + self.inserts_match_function = match_cb + self.deletes_match_function = match_cb + # Run the event loop until the correct notification arrives + try: + self.loop.run_checked() + except GraphUpdateTimeoutException: + raise GraphUpdateTimeoutException( + "Timeout waiting for property change, subject %i property %s (%i)" % (subject_id, property_uri, property_id)) + self.inserts_match_function = None + self.deletes_match_function = None + self.class_to_track = None + + # Note: The methods below call the tracker-store D-Bus API directly. This + # is useful for testing this API surface, but we recommand that all regular + # applications use libtracker-sparql library to talk to the database. + + def query(self, query, timeout=5000, **kwargs): + return self.resources.SparqlQuery('(s)', query, timeout=timeout, **kwargs) + + def update(self, update_sparql, timeout=5000, **kwargs): + return self.resources.SparqlUpdate('(s)', update_sparql, timeout=timeout, **kwargs) + + def load(self, ttl_uri, timeout=5000, **kwargs): + return self.resources.Load('(s)', ttl_uri, timeout=timeout, **kwargs) + + def batch_update(self, update_sparql, **kwargs): + return self.resources.BatchSparqlUpdate('(s)', update_sparql, **kwargs) + + def batch_commit(self, **kwargs): + return self.resources.BatchCommit(**kwargs) + + def backup(self, backup_file, **kwargs): + return self.backup_iface.Save('(s)', backup_file, **kwargs) + + def restore(self, backup_file, **kwargs): + return self.backup_iface.Restore('(s)', backup_file, **kwargs) + + def get_stats(self, **kwargs): + return self.stats_iface.Get(**kwargs) + + def get_tracker_iface(self): + return self.resources + + def count_instances(self, ontology_class): + QUERY = """ + SELECT COUNT(?u) WHERE { + ?u a %s . + } + """ + result = self.resources.SparqlQuery('(s)', QUERY % (ontology_class)) + + if (len(result) == 1): + return int(result[0][0]) + else: + return -1 + + def get_resource_id_by_uri(self, uri): + """ + Get the internal ID for a given resource, identified by URI. + """ + result = self.query( + 'SELECT tracker:id(%s) WHERE { }' % uri) + if len(result) == 1: + return int(result[0][0]) + elif len(result) == 0: + raise Exception("No entry for resource %s" % uri) + else: + raise Exception("Multiple entries for resource %s" % uri) + + # FIXME: rename to get_resource_id_by_nepomuk_url !! + def get_resource_id(self, url): + """ + Get the internal ID for a given resource, identified by URL. + """ + result = self.query( + 'SELECT tracker:id(?r) WHERE { ?r nie:url "%s" }' % url) + if len(result) == 1: + return int(result[0][0]) + elif len(result) == 0: + raise Exception("No entry for resource %s" % url) + else: + raise Exception("Multiple entries for resource %s" % url) + + def ask(self, ask_query): + assert ask_query.strip().startswith("ASK") + result = self.query(ask_query) + assert len(result) == 1 + if result[0][0] == "true": + return True + elif result[0][0] == "false": + return False + else: + raise Exception("Something fishy is going on") diff --git a/utils/trackertestutils/mainloop.py b/utils/trackertestutils/mainloop.py new file mode 100644 index 000000000..1e7a46c87 --- /dev/null +++ b/utils/trackertestutils/mainloop.py @@ -0,0 +1,58 @@ +# Copyright (C) 2018, Sam Thursfield <sam@afuera.me.uk> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + + +from gi.repository import GLib + +import sys + + +class MainLoop(): + '''Wrapper for GLib.MainLoop that propagates any unhandled exceptions. + + PyGObject doesn't seem to provide any help with propagating exceptions from + the GLib main loop to the main Python execution context. The default + behaviour is to print a message and continue, which is useless for tests as + it means tests appear to pass when in fact they are broken. + + ''' + + def __init__(self): + self._loop = GLib.MainLoop.new(None, 0) + + def quit(self): + self._loop.quit() + + def run_checked(self): + '''Run the loop until quit(), then raise any unhandled exception.''' + self._exception = None + + old_hook = sys.excepthook + + def new_hook(etype, evalue, etb): + self._loop.quit() + self._exception = evalue + old_hook(etype, evalue, etb) + + try: + sys.excepthook = new_hook + self._loop.run() + finally: + sys.excepthook = old_hook + + if self._exception: + raise self._exception diff --git a/utils/trackertestutils/meson.build b/utils/trackertestutils/meson.build new file mode 100644 index 000000000..7806ee210 --- /dev/null +++ b/utils/trackertestutils/meson.build @@ -0,0 +1,8 @@ +sources = [ + '__init__.py', + 'dconf.py', + 'helpers.py', +] + +install_data(sources, + install_dir: join_paths(tracker_internal_libs_dir, 'trackertestutils')) |