diff options
| -rw-r--r-- | ceilometer/alarm/storage/impl_hbase.py | 5 | ||||
| -rw-r--r-- | ceilometer/objectstore/swift_middleware.py | 11 | ||||
| -rw-r--r-- | ceilometer/storage/hbase/utils.py | 23 | ||||
| -rw-r--r-- | ceilometer/storage/impl_hbase.py | 6 | ||||
| -rw-r--r-- | ceilometer/tests/objectstore/test_swift_middleware.py | 21 | ||||
| -rw-r--r-- | doc/source/architecture.rst | 7 | ||||
| -rw-r--r-- | doc/source/configuration.rst | 28 | ||||
| -rw-r--r-- | doc/source/install/development.rst | 30 | ||||
| -rw-r--r-- | requirements-py3.txt | 22 | ||||
| -rw-r--r-- | requirements.txt | 30 | ||||
| -rw-r--r-- | test-requirements-py3.txt | 4 | ||||
| -rw-r--r-- | test-requirements.txt | 4 |
12 files changed, 139 insertions, 52 deletions
diff --git a/ceilometer/alarm/storage/impl_hbase.py b/ceilometer/alarm/storage/impl_hbase.py index 27311c66..8a50bcae 100644 --- a/ceilometer/alarm/storage/impl_hbase.py +++ b/ceilometer/alarm/storage/impl_hbase.py @@ -98,9 +98,10 @@ class Connection(base.Connection): self.conn_pool = self._get_connection_pool(opts) def upgrade(self): + tables = [self.ALARM_HISTORY_TABLE, self.ALARM_TABLE] + column_families = {'f': dict()} with self.conn_pool.connection() as conn: - conn.create_table(self.ALARM_TABLE, {'f': dict()}) - conn.create_table(self.ALARM_HISTORY_TABLE, {'f': dict()}) + hbase_utils.create_tables(conn, tables, column_families) def clear(self): LOG.debug(_('Dropping HBase schema...')) diff --git a/ceilometer/objectstore/swift_middleware.py b/ceilometer/objectstore/swift_middleware.py index c655ae5a..3c5fa126 100644 --- a/ceilometer/objectstore/swift_middleware.py +++ b/ceilometer/objectstore/swift_middleware.py @@ -43,6 +43,7 @@ from __future__ import absolute_import import logging from oslo.utils import timeutils +import six from ceilometer.openstack.common import context from ceilometer import pipeline @@ -145,8 +146,14 @@ class CeilometerMiddleware(object): def publish_sample(self, env, bytes_received, bytes_sent): path = env['PATH_INFO'] method = env['REQUEST_METHOD'] - headers = dict((header.strip('HTTP_'), env[header]) for header - in env if header.startswith('HTTP_')) + headers = {} + for header in env: + if header.startswith('HTTP_') and env[header]: + key = header.strip('HTTP_') + if isinstance(env[header], six.text_type): + headers[key] = env[header].encode('utf-8') + else: + headers[key] = str(env[header]) try: container = obj = None diff --git a/ceilometer/storage/hbase/utils.py b/ceilometer/storage/hbase/utils.py index 4a2a0b08..1fe4a3a4 100644 --- a/ceilometer/storage/hbase/utils.py +++ b/ceilometer/storage/hbase/utils.py @@ -17,9 +17,14 @@ import datetime import json import bson.json_util +from happybase.hbase import ttypes +from ceilometer.openstack.common.gettextutils import _ +from ceilometer.openstack.common import log from ceilometer import utils +LOG = log.getLogger(__name__) + EVENT_TRAIT_TYPES = {'none': 0, 'string': 1, 'integer': 2, 'float': 3, 'datetime': 4} OP_SIGN = {'eq': '=', 'lt': '<', 'le': '<=', 'ne': '!=', 'gt': '>', 'ge': '>='} @@ -416,3 +421,21 @@ def object_hook(dct): dt = bson.json_util.object_hook(dct) return dt.replace(tzinfo=None) return bson.json_util.object_hook(dct) + + +def create_tables(conn, tables, column_families): + for table in tables: + try: + conn.create_table(table, column_families) + except ttypes.AlreadyExists: + if conn.table_prefix: + table = ("%(table_prefix)s" + "%(separator)s" + "%(table_name)s" % + dict(table_prefix=conn.table_prefix, + separator=conn.table_prefix_separator, + table_name=table)) + + LOG.warn(_("Cannot create table %(table_name)s " + "it already exists. Ignoring error") + % {'table_name': table})
\ No newline at end of file diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index 3899f482..4a3a4e6f 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -161,10 +161,10 @@ class Connection(base.Connection): self.conn_pool = self._get_connection_pool(opts) def upgrade(self): + tables = [self.RESOURCE_TABLE, self.METER_TABLE, self.EVENT_TABLE] + column_families = {'f': dict(max_versions=1)} with self.conn_pool.connection() as conn: - conn.create_table(self.RESOURCE_TABLE, {'f': dict(max_versions=1)}) - conn.create_table(self.METER_TABLE, {'f': dict(max_versions=1)}) - conn.create_table(self.EVENT_TABLE, {'f': dict(max_versions=1)}) + hbase_utils.create_tables(conn, tables, column_families) def clear(self): LOG.debug(_('Dropping HBase schema...')) diff --git a/ceilometer/tests/objectstore/test_swift_middleware.py b/ceilometer/tests/objectstore/test_swift_middleware.py index 9021d937..fb756d78 100644 --- a/ceilometer/tests/objectstore/test_swift_middleware.py +++ b/ceilometer/tests/objectstore/test_swift_middleware.py @@ -246,6 +246,27 @@ class TestSwiftMiddleware(tests_base.BaseTestCase): data.resource_metadata['http_header_x_var2']) self.assertFalse('http_header_x_var3' in data.resource_metadata) + def test_metadata_headers_unicode(self): + app = swift_middleware.CeilometerMiddleware(FakeApp(), { + 'metadata_headers': 'unicode' + }) + uni = u'\xef\xbd\xa1\xef\xbd\xa5' + req = FakeRequest('/1.0/account/container', + environ={'REQUEST_METHOD': 'GET'}, + headers={'UNICODE': uni}) + list(app(req.environ, self.start_response)) + samples = self.pipeline_manager.pipelines[0].samples + self.assertEqual(2, len(samples)) + data = samples[0] + http_headers = [k for k in data.resource_metadata.keys() + if k.startswith('http_header_')] + self.assertEqual(1, len(http_headers)) + self.assertEqual('1.0', data.resource_metadata['version']) + self.assertEqual('container', data.resource_metadata['container']) + self.assertIsNone(data.resource_metadata['object']) + self.assertEqual(uni.encode('utf-8'), + data.resource_metadata['http_header_unicode']) + def test_metadata_headers_on_not_existing_header(self): app = swift_middleware.CeilometerMiddleware(FakeApp(), { 'metadata_headers': 'x-var3' diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst index f666251a..aaa9f094 100644 --- a/doc/source/architecture.rst +++ b/doc/source/architecture.rst @@ -349,9 +349,10 @@ agent daemon is configured to run one or more *pollster* plugins using the ``ceilometer.poll.central`` namespace. The agents periodically asks each pollster for instances of -``Counter`` objects. The agent framework converts the Counters to -metering messages, which it then signs and transmits on the metering -message bus. +``Sample`` objects. The agent framework then publishes the Samples using +the publishers defined in the pipeline configuration. For example, +the ``rpc`` publisher converts the Sample to metering messages, which it +then signs and transmits on the metering message bus. The pollster plugins do not communicate with the message bus directly, unless it is necessary to do so in order to collect the information diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 464aa96f..972ac1da 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -408,6 +408,8 @@ The chain definition looks like the following:: - 'meter filter' resources: - 'list of resource URLs' + discovery: + - 'list of discoverers' sinks - 'sink name' sinks: @@ -456,11 +458,33 @@ whose *meters* parameter matches the plugin's meter name. That is, the matching source sections are combined by union, not intersection, of the prescribed time series. -The optional *resources* section of a pipeline source allows a static -list of resource URLs to be to be configured. An amalgamated list of all +The optional *resources* section of a pipeline source allows a list of +static resource URLs to be configured. An amalgamated list of all statically configured resources for a set of pipeline sources with a common interval is passed to individual pollsters matching those pipelines. +The optional *discovery* section of a pipeline source contains the list of +discoverers. These discoverers can be used to dynamically discover the +resources to be polled by the pollsters defined in this pipeline. The name +of the discoverers should be the same as the related names of plugins in +setup.cfg. + +If *resources* or *discovery* section is not set, the default value would +be an empty list. If both *resources* and *discovery* are set, the final +resources passed to the pollsters will be the combination of the dynamic +resources returned by the discoverers and the static resources defined +in the *resources* section. If there are some duplications between the +resources returned by the discoverers and those defined in the *resources* +section, the duplication will be removed before passing those resources +to the pollsters. + +There are three ways a pollster can get a list of resources to poll, as the +following in descending order of precedence: + + 1. From the per-pipeline configured discovery and/or static resources. + 2. From the per-pollster default discovery. + 3. From the per-agent default discovery. + The *transformers* section of a pipeline sink provides the possibility to add a list of transformer definitions. The names of the transformers should be the same as the names of the related extensions in setup.cfg. For a more detailed diff --git a/doc/source/install/development.rst b/doc/source/install/development.rst index 1d45bcea..5addf61d 100644 --- a/doc/source/install/development.rst +++ b/doc/source/install/development.rst @@ -34,7 +34,15 @@ Configuring Devstack .. index:: double: installing; devstack -1. Create a ``localrc`` file as input to devstack. +1. Create a ``local.conf`` file as input to devstack. + + .. note:: + + ``local.conf`` replaces the former configuration file called ``localrc``. + If you used localrc before, remove it to switch to using the new file. + For further information see the `localrc description page + <http://devstack.org/localrc.html>`_ or `devstack configuration + <http://devstack.org/configuration.html>`_. 2. Ceilometer makes extensive use of the messaging bus, but has not yet been tested with ZeroMQ. We recommend using Rabbit or qpid for @@ -50,19 +58,21 @@ Configuring Devstack notification_driver=cinder.openstack.common.notifier.rpc_notifier 5. The ceilometer services are not enabled by default, so they must be - enabled in ``localrc`` before running ``stack.sh``. + enabled in ``local.conf`` before running ``stack.sh``. + + This example ``local.conf`` file shows all of the settings required for + ceilometer:: -This example ``localrc`` file shows all of the settings required for -ceilometer:: + [[local|localrc]] - # Enable the ceilometer metering services - enable_service ceilometer-acompute ceilometer-acentral ceilometer-anotification ceilometer-collector + # Enable the ceilometer metering services + enable_service ceilometer-acompute ceilometer-acentral ceilometer-anotification ceilometer-collector - # Enable the ceilometer alarming services - enable_service ceilometer-alarm-evaluator,ceilometer-alarm-notifier + # Enable the ceilometer alarming services + enable_service ceilometer-alarm-evaluator,ceilometer-alarm-notifier - # Enable the ceilometer api services - enable_service ceilometer-api + # Enable the ceilometer api services + enable_service ceilometer-api 6. If you use Data Processing (Sahara) service at your DevStack, it will generate notifications by default. diff --git a/requirements-py3.txt b/requirements-py3.txt index 93ce70c2..eed3775e 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -6,35 +6,35 @@ alembic>=0.6.4 anyjson>=0.3.3 argparse croniter>=0.3.4 # MIT License -eventlet>=0.13.0 +eventlet>=0.15.1 iso8601>=0.1.9 jsonpath-rw>=1.2.0,<2.0 jsonschema>=2.0.0,<3.0.0 lockfile>=0.8 lxml>=2.3 msgpack-python>=0.4.0 -netaddr>=0.7.6 -oslo.config>=1.4.0.0a3 -oslo.db>=0.4.0 +netaddr>=0.7.12 +oslo.config>=1.4.0 # Apache-2.0 +oslo.db>=1.0.0 # Apache-2.0 PasteDeploy>=1.5.0 pbr>=0.6,!=0.7,<1.0 pecan>=0.5.0 posix_ipc -oslo.messaging>=1.3.0 -oslo.utils>=0.2.0 +oslo.messaging>=1.4.0 +oslo.utils>=1.0.0 # Apache-2.0 pysnmp>=4.2.1,<5.0.0 python-ceilometerclient>=1.0.6 -python-glanceclient>=0.13.1 +python-glanceclient>=0.14.0 python-keystoneclient>=0.10.0 python-neutronclient>=2.3.6,<3 -python-novaclient>=2.17.0 +python-novaclient>=2.18.0 python-swiftclient>=2.2.0 pytz>=2010h PyYAML>=3.1.0 -requests>=1.2.1 +requests>=1.2.1,!=2.4.0 six>=1.7.0 SQLAlchemy>=0.8.4,<=0.8.99,>=0.9.7,<=0.9.99 -sqlalchemy-migrate>=0.9.1 -stevedore>=0.14 +sqlalchemy-migrate>=0.9.1,!=0.9.2 +stevedore>=1.0.0 # Apache-2.0 WebOb>=1.2.3 WSME>=0.6 diff --git a/requirements.txt b/requirements.txt index 1663de1f..4da6a0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ alembic>=0.6.4 anyjson>=0.3.3 argparse croniter>=0.3.4 # MIT License -eventlet>=0.13.0 +eventlet>=0.15.1 happybase>=0.5,!=0.7 iso8601>=0.1.9 jsonpath-rw>=1.2.0,<2.0 @@ -15,33 +15,33 @@ keystonemiddleware>=1.0.0 lockfile>=0.8 lxml>=2.3 msgpack-python>=0.4.0 -netaddr>=0.7.6 +netaddr>=0.7.12 ordereddict -oslo.db>=0.4.0 -oslo.config>=1.4.0.0a3 -oslo.rootwrap>=1.3.0.0a1 -oslo.vmware>=0.5 # Apache-2.0 +oslo.db>=1.0.0 # Apache-2.0 +oslo.config>=1.4.0 # Apache-2.0 +oslo.rootwrap>=1.3.0 +oslo.vmware>=0.6.0 # Apache-2.0 PasteDeploy>=1.5.0 pbr>=0.6,!=0.7,<1.0 pecan>=0.5.0 posix_ipc -oslo.messaging>=1.4.0.0a3 -oslo.serialization>=0.1.0 -oslo.utils>=0.2.0 # Apache-2.0 +oslo.messaging>=1.4.0 +oslo.serialization>=1.0.0 # Apache-2.0 +oslo.utils>=1.0.0 # Apache-2.0 pysnmp>=4.2.1,<5.0.0 python-ceilometerclient>=1.0.6 -python-glanceclient>=0.13.1 +python-glanceclient>=0.14.0 python-keystoneclient>=0.10.0 python-neutronclient>=2.3.6,<3 -python-novaclient>=2.17.0 +python-novaclient>=2.18.0 python-swiftclient>=2.2.0 pytz>=2010h PyYAML>=3.1.0 -requests>=1.2.1 +requests>=1.2.1,!=2.4.0 six>=1.7.0 SQLAlchemy>=0.8.4,<=0.8.99,>=0.9.7,<=0.9.99 -sqlalchemy-migrate>=0.9.1 -stevedore>=0.14 -tooz>=0.3 +sqlalchemy-migrate>=0.9.1,!=0.9.2 +stevedore>=1.0.0 # Apache-2.0 +tooz>=0.3 # Apache-2.0 WebOb>=1.2.3 WSME>=0.6 diff --git a/test-requirements-py3.txt b/test-requirements-py3.txt index 766b9b8d..f59ee530 100644 --- a/test-requirements-py3.txt +++ b/test-requirements-py3.txt @@ -12,8 +12,8 @@ httplib2>=0.7.5 mock>=1.0 mox>=0.5.3 # Docs Requirements -oslosphinx>=2.2.0.0a2 -oslotest +oslosphinx>=2.2.0 # Apache-2.0 +oslotest>=1.1.0 # Apache-2.0 pymongo>=2.5 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,<1.3 diff --git a/test-requirements.txt b/test-requirements.txt index 3cb82f7c..1fb07126 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,8 +13,8 @@ mock>=1.0 mox>=0.5.3 MySQL-python # Docs Requirements -oslosphinx>=2.2.0.0a2 -oslotest +oslosphinx>=2.2.0 # Apache-2.0 +oslotest>=1.1.0 # Apache-2.0 pymongo>=2.5 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,<1.3 |