Add a redis driver

Change-Id: I64768e282d5f3f32d9d79d15f59413d42df95037

6 files changed, 535 insertions, 2 deletions

diff --git a/doc/source/collectors.rst b/doc/source/collectors.rst
new file mode 100644
index 0000000..2bb7bdf
--- /dev/null
+++ b/doc/source/collectors.rst
@@ -0,0 +1,39 @@
+==========
+Collectors
+==========
+
+There are a number of drivers to support different collector backends:
+
+Redis:
+------
+  * Overview
+        The Redis driver allows profiling data to be collected into a redis
+        database instance. The traces are stored as key-value pairs where the
+        key is a string built using trace ids and timestamps and the values
+        are JSON strings containing the trace information. A second driver is
+        included to use Redis Sentinel in addition to single node Redis.
+
+  * Capabilities:
+        * Write trace data to the database.
+
+        * Query Traces in database: This allows for pulling trace data
+          querying on the keys used to save the data in the database.
+
+        * Generate a report based on the traces stored in the database.
+
+        * Supports use of Redis Sentinel for robustness.
+
+  * Usage:
+        The driver is used by OSProfiler when using a connection-string URL
+        of the form redis://<hostname>:<port>. To use the Sentinel version
+        use a connection-string of the form redissentinel://<hostname>:<port>
+
+  * Configuration:
+        * No config changes are required by for the base Redis driver.
+
+        * There are two configuration options for the Redis Sentinel driver:
+            * socket_timeout: specifies the sentinel connection socket timeout
+              value. Defaults to: 0.1 seconds
+
+            * sentinel_service_name: The name of the Sentinel service to use.
+              Defaults to: "mymaster"
\ No newline at end of file
diff --git a/osprofiler/drivers/__init__.py b/osprofiler/drivers/__init__.py
index c120b7e..f208582 100644
--- a/osprofiler/drivers/__init__.py
+++ b/osprofiler/drivers/__init__.py
@@ -3,3 +3,4 @@ from osprofiler.drivers import ceilometer  # noqa
 from osprofiler.drivers import elasticsearch_driver  # noqa
 from osprofiler.drivers import messaging  # noqa
 from osprofiler.drivers import mongodb  # noqa
+from osprofiler.drivers import redis_driver  # noqa
diff --git a/osprofiler/drivers/redis_driver.py b/osprofiler/drivers/redis_driver.py
new file mode 100644
index 0000000..f3f5812
--- /dev/null
+++ b/osprofiler/drivers/redis_driver.py
@@ -0,0 +1,137 @@
+# Copyright 2016 Mirantis Inc.
+# Copyright 2016 IBM Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+import six.moves.urllib.parse as parser
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class Redis(base.Driver):
+    def __init__(self, connection_str, db=0, project=None,
+                 service=None, host=None, **kwargs):
+        """Redis driver for OSProfiler."""
+
+        super(Redis, self).__init__(connection_str, project=project,
+                                    service=service, host=host)
+        try:
+            from redis import StrictRedis
+        except ImportError:
+            raise exc.CommandError(
+                "To use this command, you should install "
+                "'redis' manually. Use command:\n "
+                "'pip install redis'.")
+
+        parsed_url = parser.urlparse(self.connection_str)
+        self.db = StrictRedis(host=parsed_url.hostname,
+                              port=parsed_url.port,
+                              db=db)
+        self.namespace = "osprofiler:"
+
+    @classmethod
+    def get_name(cls):
+        return "redis"
+
+    def notify(self, info):
+        """Send notifications to Redis.
+
+        :param info:  Contains information about trace element.
+                      In payload dict there are always 3 ids:
+                      "base_id" - uuid that is common for all notifications
+                                  related to one trace. Used to simplify
+                                  retrieving of all trace elements from
+                                  Redis.
+                      "parent_id" - uuid of parent element in trace
+                      "trace_id" - uuid of current element in trace
+
+                      With parent_id and trace_id it's quite simple to build
+                      tree of trace elements, which simplify analyze of trace.
+
+        """
+        data = info.copy()
+        data["project"] = self.project
+        data["service"] = self.service
+        key = self.namespace + data["base_id"] + "_" + data["trace_id"] + "_" + \
+            data["timestamp"]
+        self.db.set(key, jsonutils.dumps(data))
+
+    def list_traces(self, query="*", fields=[]):
+        """Returns array of all base_id fields that match the given criteria
+
+        :param query: string that specifies the query criteria
+        :param fields: iterable of strings that specifies the output fields
+        """
+        for base_field in ["base_id", "timestamp"]:
+            if base_field not in fields:
+                fields.append(base_field)
+        ids = self.db.scan_iter(match=self.namespace + query)
+        traces = [jsonutils.loads(self.db.get(i)) for i in ids]
+        result = []
+        for trace in traces:
+            result.append({key: value for key, value in trace.iteritems()
+                           if key in fields})
+        return result
+
+    def get_report(self, base_id):
+        """Retrieves and parses notification from Redis.
+
+        :param base_id: Base id of trace elements.
+        """
+        for key in self.db.scan_iter(match=self.namespace + base_id + "*"):
+            data = self.db.get(key)
+            n = jsonutils.loads(data)
+            trace_id = n["trace_id"]
+            parent_id = n["parent_id"]
+            name = n["name"]
+            project = n["project"]
+            service = n["service"]
+            host = n["info"]["host"]
+            timestamp = n["timestamp"]
+
+            self._append_results(trace_id, parent_id, name, project, service,
+                                 host, timestamp, n)
+
+        return self._parse_results()
+
+
+class RedisSentinel(Redis, base.Driver):
+    def __init__(self, connection_str, db=0, project=None,
+                 service=None, host=None, conf=cfg.CONF, **kwargs):
+        """Redis driver for OSProfiler."""
+
+        super(RedisSentinel, self).__init__(connection_str, project=project,
+                                            service=service, host=host)
+        try:
+            from redis.sentinel import Sentinel
+        except ImportError:
+            raise exc.CommandError(
+                "To use this command, you should install "
+                "'redis' manually. Use command:\n "
+                "'pip install redis'.")
+
+        self.conf = conf
+        socket_timeout = self.conf.profiler.socket_timeout
+        parsed_url = parser.urlparse(self.connection_str)
+        sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))],
+                            socket_timeout=socket_timeout)
+        self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name,
+                                      socket_timeout=socket_timeout)
+
+    @classmethod
+    def get_name(cls):
+        return "redissentinel"
diff --git a/osprofiler/opts.py b/osprofiler/opts.py
index b7c7d5b..36b942f 100644
--- a/osprofiler/opts.py
+++ b/osprofiler/opts.py
@@ -117,6 +117,23 @@ Elasticsearch splits large requests in batches. This parameter defines
 maximum size of each batch (for example: es_scroll_size=10000).
 """)
 
+_socket_timeout_opt = cfg.FloatOpt(
+    "socket_timeout",
+    default=0.1,
+    help="""
+Redissentinel provides a timeout option on the connections.
+This parameter defines that timeout (for example: socket_timeout=0.1).
+""")
+
+_sentinel_service_name_opt = cfg.StrOpt(
+    "sentinel_service_name",
+    default="mymaster",
+    help="""
+Redissentinel uses a service name to identify a master redis service.
+This parameter defines the name (for example:
+sentinal_service_name=mymaster).
+""")
+
 
 _PROFILER_OPTS = [
     _enabled_opt,
@@ -125,7 +142,9 @@ _PROFILER_OPTS = [
     _connection_string_opt,
     _es_doc_type_opt,
     _es_scroll_time_opt,
-    _es_scroll_size_opt
+    _es_scroll_size_opt,
+    _socket_timeout_opt,
+    _sentinel_service_name_opt
 ]
 
 cfg.CONF.register_opts(_PROFILER_OPTS, group=_profiler_opt_group)
@@ -133,7 +152,8 @@ cfg.CONF.register_opts(_PROFILER_OPTS, group=_profiler_opt_group)
 
 def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None,
                  connection_string=None, es_doc_type=None,
-                 es_scroll_time=None, es_scroll_size=None):
+                 es_scroll_time=None, es_scroll_size=None,
+                 socket_timeout=None, sentinel_service_name=None):
     conf.register_opts(_PROFILER_OPTS, group=_profiler_opt_group)
 
     if enabled is not None:
@@ -162,6 +182,14 @@ def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None,
         conf.set_default("es_scroll_size", es_scroll_size,
                          group=_profiler_opt_group.name)
 
+    if socket_timeout is not None:
+        conf.set_default("socket_timeout", socket_timeout,
+                         group=_profiler_opt_group.name)
+
+    if sentinel_service_name is not None:
+        conf.set_default("sentinel_service_name", sentinel_service_name,
+                         group=_profiler_opt_group.name)
+
 
 def is_trace_enabled(conf=None):
     if conf is None:
diff --git a/osprofiler/tests/drivers/test_redis_driver.py b/osprofiler/tests/drivers/test_redis_driver.py
new file mode 100644
index 0000000..9fbc2e8
--- /dev/null
+++ b/osprofiler/tests/drivers/test_redis_driver.py
@@ -0,0 +1,325 @@
+# Copyright 2016 Mirantis Inc.
+# Copyright 2016 IBM Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+from oslo_serialization import jsonutils
+
+from osprofiler.drivers.redis_driver import Redis
+from osprofiler.tests import test
+
+
+class RedisParserTestCase(test.TestCase):
+    def setUp(self):
+        super(RedisParserTestCase, self).setUp()
+        self.redisdb = Redis("redis://localhost:6379")
+
+    def test_build_empty_tree(self):
+        self.assertEqual([], self.redisdb._build_tree({}))
+
+    def test_build_complex_tree(self):
+        test_input = {
+            "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+            "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+            "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+            "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+            "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+            "113": {"parent_id": "11", "trace_id": "113",
+                    "info": {"started": 3}},
+            "112": {"parent_id": "11", "trace_id": "112",
+                    "info": {"started": 2}},
+            "114": {"parent_id": "11", "trace_id": "114",
+                    "info": {"started": 5}}
+        }
+
+        expected_output = [
+            {
+                "parent_id": "0",
+                "trace_id": "1",
+                "info": {"started": 0},
+                "children": [
+                    {
+                        "parent_id": "1",
+                        "trace_id": "11",
+                        "info": {"started": 1},
+                        "children": [
+                            {"parent_id": "11", "trace_id": "112",
+                             "info": {"started": 2}, "children": []},
+                            {"parent_id": "11", "trace_id": "113",
+                             "info": {"started": 3}, "children": []},
+                            {"parent_id": "11", "trace_id": "114",
+                             "info": {"started": 5}, "children": []}
+                        ]
+                    }
+                ]
+            },
+            {
+                "parent_id": "0",
+                "trace_id": "2",
+                "info": {"started": 1},
+                "children": [
+                    {"parent_id": "2", "trace_id": "21",
+                     "info": {"started": 6}, "children": []},
+                    {"parent_id": "2", "trace_id": "22",
+                     "info": {"started": 7}, "children": []}
+                ]
+            }
+        ]
+
+        result = self.redisdb._build_tree(test_input)
+        self.assertEqual(expected_output, result)
+
+    def test_get_report_empty(self):
+        self.redisdb.db = mock.MagicMock()
+        self.redisdb.db.scan_iter.return_value = []
+
+        expected = {
+            "info": {
+                "name": "total",
+                "started": 0,
+                "finished": None
+            },
+            "children": [],
+            "stats": {},
+        }
+
+        base_id = "10"
+        self.assertEqual(expected, self.redisdb.get_report(base_id))
+
+    def test_get_report(self):
+        self.redisdb.db = mock.MagicMock()
+        result_elements = [
+            {
+                "info": {
+                    "project": None,
+                    "host": "ubuntu",
+                    "request": {
+                        "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+                        "scheme": "http",
+                        "method": "POST",
+                        "query": ""
+                    },
+                    "service": None
+                },
+                "name": "wsgi-start",
+                "service": "main",
+                "timestamp": "2015-12-23T14:02:22.338776",
+                "trace_id": "06320327-2c2c-45ae-923a-515de890276a",
+                "project": "keystone",
+                "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+            },
+
+            {
+                "info": {
+                    "project": None,
+                    "host": "ubuntu",
+                    "service": None
+                },
+                "name": "wsgi-stop",
+                "service": "main",
+                "timestamp": "2015-12-23T14:02:22.380405",
+                "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf",
+                "project": "keystone",
+                "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+            },
+
+            {
+                "info": {
+                    "project": None,
+                    "host": "ubuntu",
+                    "db": {
+                        "params": {
+
+                            },
+                        "statement": "SELECT 1"
+                    },
+                    "service": None
+                },
+                "name": "db-start",
+                "service": "main",
+                "timestamp": "2015-12-23T14:02:22.395365",
+                "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+                "project": "keystone",
+                "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+                "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+            },
+
+            {
+                "info": {
+                    "project": None,
+                    "host": "ubuntu",
+                    "service": None
+                },
+                "name": "db-stop",
+                "service": "main",
+                "timestamp": "2015-12-23T14:02:22.415486",
+                "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+                "project": "keystone",
+                "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+                "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+            },
+
+            {
+                "info": {
+                    "project": None,
+                    "host": "ubuntu",
+                    "request": {
+                        "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+                        "scheme": "http",
+                        "method": "GET",
+                        "query": ""
+                    },
+                    "service": None
+                },
+                "name": "wsgi-start",
+                "service": "main",
+                "timestamp": "2015-12-23T14:02:22.427444",
+                "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b",
+                "project": "keystone",
+                "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+            }]
+        results = {result["base_id"] + "_" + result["trace_id"] +
+                   "_" + result["timestamp"]: result
+                   for result in result_elements}
+
+        expected = {"children": [{"children": [{
+            "children": [],
+            "info": {"finished": 76,
+                     "host": "ubuntu",
+                     "meta.raw_payload.db-start": {
+                         "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                         "info": {"db": {"params": {},
+                                         "statement": "SELECT 1"},
+                                  "host": "ubuntu",
+                                  "project": None,
+                                  "service": None},
+                         "name": "db-start",
+                         "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+                         "project": "keystone",
+                         "service": "main",
+                         "timestamp": "2015-12-23T14:02:22.395365",
+                         "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+                     "meta.raw_payload.db-stop": {
+                         "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                         "info": {"host": "ubuntu",
+                                  "project": None,
+                                  "service": None},
+                         "name": "db-stop",
+                         "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+                         "project": "keystone",
+                         "service": "main",
+                         "timestamp": "2015-12-23T14:02:22.415486",
+                         "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+                     "name": "db",
+                     "project": "keystone",
+                     "service": "main",
+                     "started": 56},
+            "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+            "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}],
+
+            "info": {"finished": 0,
+                     "host": "ubuntu",
+                     "meta.raw_payload.wsgi-start": {
+                         "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                         "info": {"host": "ubuntu",
+                                  "project": None,
+                                  "request": {"method": "POST",
+                                              "path": "/v2/a322b5049d224a90bf8"
+                                              "786c644409400/volumes",
+                                              "query": "",
+                                              "scheme": "http"},
+                                  "service": None},
+                         "name": "wsgi-start",
+                         "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                         "project": "keystone",
+                         "service": "main",
+                         "timestamp": "2015-12-23T14:02:22.338776",
+                         "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+                     "name": "wsgi",
+                     "project": "keystone",
+                     "service": "main",
+                     "started": 0},
+            "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+            "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+
+            {"children": [],
+             "info": {"finished": 41,
+                      "host": "ubuntu",
+                      "meta.raw_payload.wsgi-stop": {
+                          "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                          "info": {"host": "ubuntu",
+                                   "project": None,
+                                   "service": None},
+                          "name": "wsgi-stop",
+                          "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                          "project": "keystone",
+                          "service": "main",
+                          "timestamp": "2015-12-23T14:02:22.380405",
+                          "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+                      "name": "wsgi",
+                      "project": "keystone",
+                      "service": "main",
+                      "started": 41},
+             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+             "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+
+            {"children": [],
+             "info": {"finished": 88,
+                      "host": "ubuntu",
+                      "meta.raw_payload.wsgi-start": {
+                          "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                          "info": {"host": "ubuntu",
+                                   "project": None,
+                                   "request": {"method": "GET",
+                                               "path": "/v2/a322b5049d224a90bf"
+                                               "8786c644409400/volumes",
+                                               "query": "",
+                                               "scheme": "http"},
+                                   "service": None},
+                          "name": "wsgi-start",
+                          "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+                          "project": "keystone",
+                          "service": "main",
+                          "timestamp": "2015-12-23T14:02:22.427444",
+                          "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"},
+                      "name": "wsgi",
+                      "project": "keystone",
+                      "service": "main",
+                      "started": 88},
+             "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+             "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
+            "info": {"finished": 88, "name": "total", "started": 0},
+            "stats": {"db": {"count": 1, "duration": 20},
+                      "wsgi": {"count": 3, "duration": 0}}}
+
+        self.redisdb.db.scan_iter.return_value = list(results.keys())
+        print(results.keys())
+
+        def side_effect(*args, **kwargs):
+            return jsonutils.dumps(results[args[0]])
+
+        self.redisdb.db.get.side_effect = side_effect
+
+        base_id = "10"
+
+        result = self.redisdb.get_report(base_id)
+
+        expected_filter = self.redisdb.namespace + "10*"
+        self.redisdb.db.scan_iter.assert_called_once_with(
+            match=expected_filter)
+        self.assertEqual(expected, result)
diff --git a/test-requirements.txt b/test-requirements.txt
index 3de1138..075dfcf 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -18,5 +18,8 @@ pymongo>=3.0.2,!=3.1 # Apache-2.0
 # Elasticsearch python client
 elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
 
+# Redis python client
+redis>=2.10.0 # MIT
+
 # Build release notes
 reno>=1.8.0 # Apache-2.0