diff options
author | BENJAMIN VANHAVERMAET <bvanhav@us.ibm.com> | 2016-09-01 11:11:49 -0500 |
---|---|---|
committer | BENJAMIN VANHAVERMAET <bvanhav@us.ibm.com> | 2016-11-07 06:35:52 -0600 |
commit | 1c6f8b60b279b053f75c63f1a66158108cdf4456 (patch) | |
tree | 13738ed066aefbe1a478aa5f1c0962140919621b | |
parent | 2a3db90cfb65fc569c4d3a9d9a19c046fc56aacf (diff) | |
download | osprofiler-1c6f8b60b279b053f75c63f1a66158108cdf4456.tar.gz |
Add a redis driver
Change-Id: I64768e282d5f3f32d9d79d15f59413d42df95037
-rw-r--r-- | doc/source/collectors.rst | 39 | ||||
-rw-r--r-- | osprofiler/drivers/__init__.py | 1 | ||||
-rw-r--r-- | osprofiler/drivers/redis_driver.py | 137 | ||||
-rw-r--r-- | osprofiler/opts.py | 32 | ||||
-rw-r--r-- | osprofiler/tests/drivers/test_redis_driver.py | 325 | ||||
-rw-r--r-- | test-requirements.txt | 3 |
6 files changed, 535 insertions, 2 deletions
diff --git a/doc/source/collectors.rst b/doc/source/collectors.rst new file mode 100644 index 0000000..2bb7bdf --- /dev/null +++ b/doc/source/collectors.rst @@ -0,0 +1,39 @@ +========== +Collectors +========== + +There are a number of drivers to support different collector backends: + +Redis: +------ + * Overview + The Redis driver allows profiling data to be collected into a redis + database instance. The traces are stored as key-value pairs where the + key is a string built using trace ids and timestamps and the values + are JSON strings containing the trace information. A second driver is + included to use Redis Sentinel in addition to single node Redis. + + * Capabilities: + * Write trace data to the database. + + * Query Traces in database: This allows for pulling trace data + querying on the keys used to save the data in the database. + + * Generate a report based on the traces stored in the database. + + * Supports use of Redis Sentinel for robustness. + + * Usage: + The driver is used by OSProfiler when using a connection-string URL + of the form redis://<hostname>:<port>. To use the Sentinel version + use a connection-string of the form redissentinel://<hostname>:<port> + + * Configuration: + * No config changes are required by for the base Redis driver. + + * There are two configuration options for the Redis Sentinel driver: + * socket_timeout: specifies the sentinel connection socket timeout + value. Defaults to: 0.1 seconds + + * sentinel_service_name: The name of the Sentinel service to use. + Defaults to: "mymaster"
\ No newline at end of file diff --git a/osprofiler/drivers/__init__.py b/osprofiler/drivers/__init__.py index c120b7e..f208582 100644 --- a/osprofiler/drivers/__init__.py +++ b/osprofiler/drivers/__init__.py @@ -3,3 +3,4 @@ from osprofiler.drivers import ceilometer # noqa from osprofiler.drivers import elasticsearch_driver # noqa from osprofiler.drivers import messaging # noqa from osprofiler.drivers import mongodb # noqa +from osprofiler.drivers import redis_driver # noqa diff --git a/osprofiler/drivers/redis_driver.py b/osprofiler/drivers/redis_driver.py new file mode 100644 index 0000000..f3f5812 --- /dev/null +++ b/osprofiler/drivers/redis_driver.py @@ -0,0 +1,137 @@ +# Copyright 2016 Mirantis Inc. +# Copyright 2016 IBM Corporation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_serialization import jsonutils +import six.moves.urllib.parse as parser + +from osprofiler.drivers import base +from osprofiler import exc + + +class Redis(base.Driver): + def __init__(self, connection_str, db=0, project=None, + service=None, host=None, **kwargs): + """Redis driver for OSProfiler.""" + + super(Redis, self).__init__(connection_str, project=project, + service=service, host=host) + try: + from redis import StrictRedis + except ImportError: + raise exc.CommandError( + "To use this command, you should install " + "'redis' manually. Use command:\n " + "'pip install redis'.") + + parsed_url = parser.urlparse(self.connection_str) + self.db = StrictRedis(host=parsed_url.hostname, + port=parsed_url.port, + db=db) + self.namespace = "osprofiler:" + + @classmethod + def get_name(cls): + return "redis" + + def notify(self, info): + """Send notifications to Redis. + + :param info: Contains information about trace element. + In payload dict there are always 3 ids: + "base_id" - uuid that is common for all notifications + related to one trace. Used to simplify + retrieving of all trace elements from + Redis. + "parent_id" - uuid of parent element in trace + "trace_id" - uuid of current element in trace + + With parent_id and trace_id it's quite simple to build + tree of trace elements, which simplify analyze of trace. + + """ + data = info.copy() + data["project"] = self.project + data["service"] = self.service + key = self.namespace + data["base_id"] + "_" + data["trace_id"] + "_" + \ + data["timestamp"] + self.db.set(key, jsonutils.dumps(data)) + + def list_traces(self, query="*", fields=[]): + """Returns array of all base_id fields that match the given criteria + + :param query: string that specifies the query criteria + :param fields: iterable of strings that specifies the output fields + """ + for base_field in ["base_id", "timestamp"]: + if base_field not in fields: + fields.append(base_field) + ids = self.db.scan_iter(match=self.namespace + query) + traces = [jsonutils.loads(self.db.get(i)) for i in ids] + result = [] + for trace in traces: + result.append({key: value for key, value in trace.iteritems() + if key in fields}) + return result + + def get_report(self, base_id): + """Retrieves and parses notification from Redis. + + :param base_id: Base id of trace elements. + """ + for key in self.db.scan_iter(match=self.namespace + base_id + "*"): + data = self.db.get(key) + n = jsonutils.loads(data) + trace_id = n["trace_id"] + parent_id = n["parent_id"] + name = n["name"] + project = n["project"] + service = n["service"] + host = n["info"]["host"] + timestamp = n["timestamp"] + + self._append_results(trace_id, parent_id, name, project, service, + host, timestamp, n) + + return self._parse_results() + + +class RedisSentinel(Redis, base.Driver): + def __init__(self, connection_str, db=0, project=None, + service=None, host=None, conf=cfg.CONF, **kwargs): + """Redis driver for OSProfiler.""" + + super(RedisSentinel, self).__init__(connection_str, project=project, + service=service, host=host) + try: + from redis.sentinel import Sentinel + except ImportError: + raise exc.CommandError( + "To use this command, you should install " + "'redis' manually. Use command:\n " + "'pip install redis'.") + + self.conf = conf + socket_timeout = self.conf.profiler.socket_timeout + parsed_url = parser.urlparse(self.connection_str) + sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))], + socket_timeout=socket_timeout) + self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name, + socket_timeout=socket_timeout) + + @classmethod + def get_name(cls): + return "redissentinel" diff --git a/osprofiler/opts.py b/osprofiler/opts.py index b7c7d5b..36b942f 100644 --- a/osprofiler/opts.py +++ b/osprofiler/opts.py @@ -117,6 +117,23 @@ Elasticsearch splits large requests in batches. This parameter defines maximum size of each batch (for example: es_scroll_size=10000). """) +_socket_timeout_opt = cfg.FloatOpt( + "socket_timeout", + default=0.1, + help=""" +Redissentinel provides a timeout option on the connections. +This parameter defines that timeout (for example: socket_timeout=0.1). +""") + +_sentinel_service_name_opt = cfg.StrOpt( + "sentinel_service_name", + default="mymaster", + help=""" +Redissentinel uses a service name to identify a master redis service. +This parameter defines the name (for example: +sentinal_service_name=mymaster). +""") + _PROFILER_OPTS = [ _enabled_opt, @@ -125,7 +142,9 @@ _PROFILER_OPTS = [ _connection_string_opt, _es_doc_type_opt, _es_scroll_time_opt, - _es_scroll_size_opt + _es_scroll_size_opt, + _socket_timeout_opt, + _sentinel_service_name_opt ] cfg.CONF.register_opts(_PROFILER_OPTS, group=_profiler_opt_group) @@ -133,7 +152,8 @@ cfg.CONF.register_opts(_PROFILER_OPTS, group=_profiler_opt_group) def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None, connection_string=None, es_doc_type=None, - es_scroll_time=None, es_scroll_size=None): + es_scroll_time=None, es_scroll_size=None, + socket_timeout=None, sentinel_service_name=None): conf.register_opts(_PROFILER_OPTS, group=_profiler_opt_group) if enabled is not None: @@ -162,6 +182,14 @@ def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None, conf.set_default("es_scroll_size", es_scroll_size, group=_profiler_opt_group.name) + if socket_timeout is not None: + conf.set_default("socket_timeout", socket_timeout, + group=_profiler_opt_group.name) + + if sentinel_service_name is not None: + conf.set_default("sentinel_service_name", sentinel_service_name, + group=_profiler_opt_group.name) + def is_trace_enabled(conf=None): if conf is None: diff --git a/osprofiler/tests/drivers/test_redis_driver.py b/osprofiler/tests/drivers/test_redis_driver.py new file mode 100644 index 0000000..9fbc2e8 --- /dev/null +++ b/osprofiler/tests/drivers/test_redis_driver.py @@ -0,0 +1,325 @@ +# Copyright 2016 Mirantis Inc. +# Copyright 2016 IBM Corporation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslo_serialization import jsonutils + +from osprofiler.drivers.redis_driver import Redis +from osprofiler.tests import test + + +class RedisParserTestCase(test.TestCase): + def setUp(self): + super(RedisParserTestCase, self).setUp() + self.redisdb = Redis("redis://localhost:6379") + + def test_build_empty_tree(self): + self.assertEqual([], self.redisdb._build_tree({})) + + def test_build_complex_tree(self): + test_input = { + "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}}, + "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}}, + "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}}, + "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}}, + "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}}, + "113": {"parent_id": "11", "trace_id": "113", + "info": {"started": 3}}, + "112": {"parent_id": "11", "trace_id": "112", + "info": {"started": 2}}, + "114": {"parent_id": "11", "trace_id": "114", + "info": {"started": 5}} + } + + expected_output = [ + { + "parent_id": "0", + "trace_id": "1", + "info": {"started": 0}, + "children": [ + { + "parent_id": "1", + "trace_id": "11", + "info": {"started": 1}, + "children": [ + {"parent_id": "11", "trace_id": "112", + "info": {"started": 2}, "children": []}, + {"parent_id": "11", "trace_id": "113", + "info": {"started": 3}, "children": []}, + {"parent_id": "11", "trace_id": "114", + "info": {"started": 5}, "children": []} + ] + } + ] + }, + { + "parent_id": "0", + "trace_id": "2", + "info": {"started": 1}, + "children": [ + {"parent_id": "2", "trace_id": "21", + "info": {"started": 6}, "children": []}, + {"parent_id": "2", "trace_id": "22", + "info": {"started": 7}, "children": []} + ] + } + ] + + result = self.redisdb._build_tree(test_input) + self.assertEqual(expected_output, result) + + def test_get_report_empty(self): + self.redisdb.db = mock.MagicMock() + self.redisdb.db.scan_iter.return_value = [] + + expected = { + "info": { + "name": "total", + "started": 0, + "finished": None + }, + "children": [], + "stats": {}, + } + + base_id = "10" + self.assertEqual(expected, self.redisdb.get_report(base_id)) + + def test_get_report(self): + self.redisdb.db = mock.MagicMock() + result_elements = [ + { + "info": { + "project": None, + "host": "ubuntu", + "request": { + "path": "/v2/a322b5049d224a90bf8786c644409400/volumes", + "scheme": "http", + "method": "POST", + "query": "" + }, + "service": None + }, + "name": "wsgi-start", + "service": "main", + "timestamp": "2015-12-23T14:02:22.338776", + "trace_id": "06320327-2c2c-45ae-923a-515de890276a", + "project": "keystone", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + + { + "info": { + "project": None, + "host": "ubuntu", + "service": None + }, + "name": "wsgi-stop", + "service": "main", + "timestamp": "2015-12-23T14:02:22.380405", + "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf", + "project": "keystone", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + + { + "info": { + "project": None, + "host": "ubuntu", + "db": { + "params": { + + }, + "statement": "SELECT 1" + }, + "service": None + }, + "name": "db-start", + "service": "main", + "timestamp": "2015-12-23T14:02:22.395365", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a", + "project": "keystone", + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + + { + "info": { + "project": None, + "host": "ubuntu", + "service": None + }, + "name": "db-stop", + "service": "main", + "timestamp": "2015-12-23T14:02:22.415486", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a", + "project": "keystone", + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }, + + { + "info": { + "project": None, + "host": "ubuntu", + "request": { + "path": "/v2/a322b5049d224a90bf8786c644409400/volumes", + "scheme": "http", + "method": "GET", + "query": "" + }, + "service": None + }, + "name": "wsgi-start", + "service": "main", + "timestamp": "2015-12-23T14:02:22.427444", + "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b", + "project": "keystone", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4" + }] + results = {result["base_id"] + "_" + result["trace_id"] + + "_" + result["timestamp"]: result + for result in result_elements} + + expected = {"children": [{"children": [{ + "children": [], + "info": {"finished": 76, + "host": "ubuntu", + "meta.raw_payload.db-start": { + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "info": {"db": {"params": {}, + "statement": "SELECT 1"}, + "host": "ubuntu", + "project": None, + "service": None}, + "name": "db-start", + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "project": "keystone", + "service": "main", + "timestamp": "2015-12-23T14:02:22.395365", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}, + "meta.raw_payload.db-stop": { + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "info": {"host": "ubuntu", + "project": None, + "service": None}, + "name": "db-stop", + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "project": "keystone", + "service": "main", + "timestamp": "2015-12-23T14:02:22.415486", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}, + "name": "db", + "project": "keystone", + "service": "main", + "started": 56}, + "parent_id": "06320327-2c2c-45ae-923a-515de890276a", + "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}], + + "info": {"finished": 0, + "host": "ubuntu", + "meta.raw_payload.wsgi-start": { + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "info": {"host": "ubuntu", + "project": None, + "request": {"method": "POST", + "path": "/v2/a322b5049d224a90bf8" + "786c644409400/volumes", + "query": "", + "scheme": "http"}, + "service": None}, + "name": "wsgi-start", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "project": "keystone", + "service": "main", + "timestamp": "2015-12-23T14:02:22.338776", + "trace_id": "06320327-2c2c-45ae-923a-515de890276a"}, + "name": "wsgi", + "project": "keystone", + "service": "main", + "started": 0}, + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "trace_id": "06320327-2c2c-45ae-923a-515de890276a"}, + + {"children": [], + "info": {"finished": 41, + "host": "ubuntu", + "meta.raw_payload.wsgi-stop": { + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "info": {"host": "ubuntu", + "project": None, + "service": None}, + "name": "wsgi-stop", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "project": "keystone", + "service": "main", + "timestamp": "2015-12-23T14:02:22.380405", + "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"}, + "name": "wsgi", + "project": "keystone", + "service": "main", + "started": 41}, + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"}, + + {"children": [], + "info": {"finished": 88, + "host": "ubuntu", + "meta.raw_payload.wsgi-start": { + "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "info": {"host": "ubuntu", + "project": None, + "request": {"method": "GET", + "path": "/v2/a322b5049d224a90bf" + "8786c644409400/volumes", + "query": "", + "scheme": "http"}, + "service": None}, + "name": "wsgi-start", + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "project": "keystone", + "service": "main", + "timestamp": "2015-12-23T14:02:22.427444", + "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}, + "name": "wsgi", + "project": "keystone", + "service": "main", + "started": 88}, + "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4", + "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}], + "info": {"finished": 88, "name": "total", "started": 0}, + "stats": {"db": {"count": 1, "duration": 20}, + "wsgi": {"count": 3, "duration": 0}}} + + self.redisdb.db.scan_iter.return_value = list(results.keys()) + print(results.keys()) + + def side_effect(*args, **kwargs): + return jsonutils.dumps(results[args[0]]) + + self.redisdb.db.get.side_effect = side_effect + + base_id = "10" + + result = self.redisdb.get_report(base_id) + + expected_filter = self.redisdb.namespace + "10*" + self.redisdb.db.scan_iter.assert_called_once_with( + match=expected_filter) + self.assertEqual(expected, result) diff --git a/test-requirements.txt b/test-requirements.txt index 3de1138..075dfcf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,5 +18,8 @@ pymongo>=3.0.2,!=3.1 # Apache-2.0 # Elasticsearch python client elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0 +# Redis python client +redis>=2.10.0 # MIT + # Build release notes reno>=1.8.0 # Apache-2.0 |