Reformat and improve kombu transport reference docs (#1264)

18 files changed, 500 insertions, 78 deletions

diff --git a/docs/reference/index.rst b/docs/reference/index.rst
index fea4dff6..bddbf2f5 100644
--- a/docs/reference/index.rst
+++ b/docs/reference/index.rst
@@ -52,7 +52,6 @@ Kombu Transports
     kombu.transport.zookeeper
     kombu.transport.filesystem
     kombu.transport.sqlalchemy
-    kombu.transport.sqlalchemy.models
     kombu.transport.SQS
     kombu.transport.SLMQ
     kombu.transport.pyro
diff --git a/docs/reference/kombu.transport.sqlalchemy.models.rst b/docs/reference/kombu.transport.sqlalchemy.models.rst
deleted file mode 100644
index f4aa3ec3..00000000
--- a/docs/reference/kombu.transport.sqlalchemy.models.rst
+++ /dev/null
@@ -1,32 +0,0 @@
-=====================================================================
- SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy.models``
-=====================================================================
-
-
-.. currentmodule:: kombu.transport.sqlalchemy.models
-
-.. automodule:: kombu.transport.sqlalchemy.models
-
-    .. contents::
-        :local:
-
-    Models
-    ------
-
-    .. autoclass:: Queue
-
-        .. autoattribute:: Queue.id
-
-        .. autoattribute:: Queue.name
-
-    .. autoclass:: Message
-
-        .. autoattribute:: Message.id
-
-        .. autoattribute:: Message.visible
-
-        .. autoattribute:: Message.sent_at
-
-        .. autoattribute:: Message.payload
-
-        .. autoattribute:: Message.version
diff --git a/docs/reference/kombu.transport.sqlalchemy.rst b/docs/reference/kombu.transport.sqlalchemy.rst
index 848110d3..afa3f14e 100644
--- a/docs/reference/kombu.transport.sqlalchemy.rst
+++ b/docs/reference/kombu.transport.sqlalchemy.rst
@@ -1,6 +1,6 @@
-===========================================================
- SQLAlchemy Transport Model - kombu.transport.sqlalchemy
-===========================================================
+=============================================================
+ SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy``
+=============================================================
 
 
 .. currentmodule:: kombu.transport.sqlalchemy
@@ -23,3 +23,36 @@
     .. autoclass:: Channel
         :members:
         :undoc-members:
+
+=====================================================================
+ SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy.models``
+=====================================================================
+
+
+.. currentmodule:: kombu.transport.sqlalchemy.models
+
+.. automodule:: kombu.transport.sqlalchemy.models
+
+    .. contents::
+        :local:
+
+    Models
+    ------
+
+    .. autoclass:: Queue
+
+        .. autoattribute:: Queue.id
+
+        .. autoattribute:: Queue.name
+
+    .. autoclass:: Message
+
+        .. autoattribute:: Message.id
+
+        .. autoattribute:: Message.visible
+
+        .. autoattribute:: Message.sent_at
+
+        .. autoattribute:: Message.payload
+
+        .. autoattribute:: Message.version
diff --git a/kombu/transport/SLMQ.py b/kombu/transport/SLMQ.py
index 489db3bd..ab23f88c 100644
--- a/kombu/transport/SLMQ.py
+++ b/kombu/transport/SLMQ.py
@@ -1,4 +1,22 @@
-"""SoftLayer Message Queue transport."""
+"""SoftLayer Message Queue transport module for kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+ *Unreviewed*
+
+Transport Options
+=================
+ *Unreviewed*
+"""
 
 import socket
 import string
diff --git a/kombu/transport/SQS.py b/kombu/transport/SQS.py
index b4a7f3c2..74cc86fd 100644
--- a/kombu/transport/SQS.py
+++ b/kombu/transport/SQS.py
@@ -81,6 +81,15 @@ as follows:
 
 For a complete list of settings you can adjust using this option see
 https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: No
+* Supports TTL: No
 """  # noqa: E501
 
 
diff --git a/kombu/transport/azureservicebus.py b/kombu/transport/azureservicebus.py
index 6cd62c44..ff251027 100644
--- a/kombu/transport/azureservicebus.py
+++ b/kombu/transport/azureservicebus.py
@@ -1,10 +1,4 @@
-"""Azure Service Bus Message Queue transport.
-
-The transport can be enabled by setting the CELERY_BROKER_URL to:
-
-```
-azureservicebus://{SAS policy name}:{SAS key}@{Service Bus Namespace}
-```
+"""Azure Service Bus Message Queue transport module for kombu.
 
 Note that the Shared Access Policy used to connect to Azure Service Bus
 requires Manage, Send and Listen claims since the broker will create new
@@ -16,6 +10,31 @@ have to be regenerated before it can be used in the connection URL.
 More information about Azure Service Bus:
 https://azure.microsoft.com/en-us/services/service-bus/
 
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    azureservicebus://SAS_POLICY_NAME:SAS_KEY@SERVICE_BUSNAMESPACE
+
+Transport Options
+=================
+
+* ``visibility_timeout``
+* ``queue_name_prefix``
+* ``wait_time_seconds``
+* ``peek_lock``
 """
 
 import string
diff --git a/kombu/transport/azurestoragequeues.py b/kombu/transport/azurestoragequeues.py
index a38aa748..76fde77c 100644
--- a/kombu/transport/azurestoragequeues.py
+++ b/kombu/transport/azurestoragequeues.py
@@ -1,17 +1,33 @@
-"""Azure Storage Queues transport.
+"""Azure Storage Queues transport module for kombu.
 
-The transport can be enabled by setting the CELERY_BROKER_URL to:
+More information about Azure Storage Queues:
+https://azure.microsoft.com/en-us/services/storage/queues/
+
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
 
-```
-azurestoragequeues://:{Storage Account Access Key}@{Storage Account Name}
-```
+Connection string has the following format:
+
+.. code-block::
+
+    azurestoragequeues://:STORAGE_ACCOUNT_ACCESS kEY@STORAGE_ACCOUNT_NAME
 
 Note that if the access key for the storage account contains a slash, it will
 have to be regenerated before it can be used in the connection URL.
 
-More information about Azure Storage Queues:
-https://azure.microsoft.com/en-us/services/storage/queues/
+Transport Options
+=================
 
+* ``queue_name_prefix``
 """
 
 from queue import Empty
diff --git a/kombu/transport/consul.py b/kombu/transport/consul.py
index d7856295..5bebd8e4 100644
--- a/kombu/transport/consul.py
+++ b/kombu/transport/consul.py
@@ -1,8 +1,30 @@
-"""Consul Transport.
+"""Consul Transport module for Kombu.
+
+Features
+========
 
 It uses Consul.io's Key/Value store to transport messages in Queues
 
 It uses python-consul for talking to Consul's HTTP API
+
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    consul://CONSUL_ADDRESS[:PORT]
+
 """
 
 import uuid
diff --git a/kombu/transport/etcd.py b/kombu/transport/etcd.py
index a58b9432..5ff7b922 100644
--- a/kombu/transport/etcd.py
+++ b/kombu/transport/etcd.py
@@ -1,8 +1,27 @@
-"""Etcd Transport.
+"""Etcd Transport module for Kombu.
 
 It uses Etcd as a store to transport messages in Queues
 
 It uses python-etcd for talking to Etcd's HTTP API
+
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    'etcd'://SERVER:PORT
+
 """
 
 import os
diff --git a/kombu/transport/filesystem.py b/kombu/transport/filesystem.py
index 5dc64162..5edaa237 100644
--- a/kombu/transport/filesystem.py
+++ b/kombu/transport/filesystem.py
@@ -1,6 +1,91 @@
-"""File-system Transport.
-
-Transport using the file-system as the message store.
+"""File-system Transport module for kombu.
+
+Transport using the file-system as the message store. Messages written to the
+queue are stored in `data_folder_in` directory and
+messages read from the queue are read from `data_folder_out` directory. Both
+directories must be created manually. Simple example:
+
+* Producer:
+
+.. code-block:: python
+
+    import kombu
+
+    conn = kombu.Connection(
+        'filesystem://', transport_options={
+            'data_folder_in': 'data_in', 'data_folder_out': 'data_out'
+        }
+    )
+    conn.connect()
+
+    test_queue = kombu.Queue('test', routing_key='test')
+
+    with conn as conn:
+        with conn.default_channel as channel:
+            producer = kombu.Producer(channel)
+            producer.publish(
+                        {'hello': 'world'},
+                        retry=True,
+                        exchange=test_queue.exchange,
+                        routing_key=test_queue.routing_key,
+                        declare=[test_queue],
+                        serializer='pickle'
+            )
+
+* Consumer:
+
+.. code-block:: python
+
+    import kombu
+
+    conn = kombu.Connection(
+        'filesystem://', transport_options={
+            'data_folder_in': 'data_out', 'data_folder_out': 'data_in'
+        }
+    )
+    conn.connect()
+
+    def callback(body, message):
+        print(body, message)
+        message.ack()
+
+    test_queue = kombu.Queue('test', routing_key='test')
+
+    with conn as conn:
+        with conn.default_channel as channel:
+            consumer = kombu.Consumer(
+                conn, [test_queue], accept=['pickle']
+            )
+            consumer.register_callback(callback)
+            with consumer:
+                conn.drain_events(timeout=1)
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+Connection string is in the following format:
+
+.. code-block::
+
+    filesystem://
+
+Transport Options
+=================
+* ``data_folder_in`` - directory where are messages stored when written
+  to queue.
+* ``data_folder_out`` - directory from which are messages read when read from
+  queue.
+* ``store_processed`` - if set to True, all processed messages are backed up to
+  ``processed_folder``.
+* ``processed_folder`` - directory where are backed up processed files.
 """
 
 import os
diff --git a/kombu/transport/memory.py b/kombu/transport/memory.py
index f408b1fe..72023e9e 100644
--- a/kombu/transport/memory.py
+++ b/kombu/transport/memory.py
@@ -1,4 +1,26 @@
-"""In-memory transport."""
+"""In-memory transport module for Kombu.
+
+Simple transport using memory for storing messages.
+Messages can be passed only between threads.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: Yes
+
+Connection String
+=================
+Connection string is in the following format:
+
+.. code-block::
+
+    memory://
+
+"""
 
 from queue import Queue
 
diff --git a/kombu/transport/mongodb.py b/kombu/transport/mongodb.py
index 2d802d61..d0e12e4a 100644
--- a/kombu/transport/mongodb.py
+++ b/kombu/transport/mongodb.py
@@ -1,7 +1,36 @@
-"""MongoDB transport.
-
-:copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
-:license: BSD, see LICENSE for more details.
+# copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
+# license: BSD, see LICENSE for more details.
+
+"""MongoDB transport module for kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
+Connection String
+=================
+ *Unreviewed*
+
+Transport Options
+=================
+
+* ``connect_timeout``,
+* ``ssl``,
+* ``ttl``,
+* ``capped_queue_size``,
+* ``default_hostname``,
+* ``default_port``,
+* ``default_database``,
+* ``messages_collection``,
+* ``routing_collection``,
+* ``broadcast_collection``,
+* ``queues_collection``,
+* ``calc_queue_size``,
 """
 
 import datetime
diff --git a/kombu/transport/pyamqp.py b/kombu/transport/pyamqp.py
index 3a2763b9..63c0a973 100644
--- a/kombu/transport/pyamqp.py
+++ b/kombu/transport/pyamqp.py
@@ -1,4 +1,38 @@
-"""Pure-Python amqp transport."""
+"""pyamqp transport module for Kombu.
+
+Pure-Python amqp transport using py-amqp library.
+
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
+Connection String
+=================
+Connection string can have the following formats:
+
+.. code-block::
+
+    amqp://[USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+    [USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+    amqp://
+
+For TLS encryption use:
+
+.. code-block::
+
+    amqps://[USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+
+Transport Options
+=================
+Transport Options are passed to constructor of underlying py-amqp
+:class:`~kombu.connection.Connection` class.
+"""
+
 
 import amqp
 
diff --git a/kombu/transport/pyro.py b/kombu/transport/pyro.py
index cddddbe8..9e86319b 100644
--- a/kombu/transport/pyro.py
+++ b/kombu/transport/pyro.py
@@ -1,14 +1,34 @@
-"""Pyro transport, and Kombu Broker daemon.
+"""Pyro transport module for kombu.
+
+Pyro transport, and Kombu Broker daemon.
 
 Requires the :mod:`Pyro4` library to be installed.
 
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+
 To use the Pyro transport with Kombu, use an url of the form:
-``pyro://localhost/kombu.broker``
+
+.. code-block::
+
+    pyro://localhost/kombu.broker
 
 The hostname is where the transport will be looking for a Pyro name server,
 which is used in turn to locate the kombu.broker Pyro service.
 This broker can be launched by simply executing this transport module directly,
 with the command: ``python -m kombu.transport.pyro``
+
+Transport Options
+=================
 """
 
 
diff --git a/kombu/transport/qpid.py b/kombu/transport/qpid.py
index 897d9ba4..cacd3489 100644
--- a/kombu/transport/qpid.py
+++ b/kombu/transport/qpid.py
@@ -1,4 +1,4 @@
-"""Qpid Transport.
+"""Qpid Transport module for kombu.
 
 `Qpid`_ transport using `qpid-python`_ as the client and `qpid-tools`_ for
 broker management.
@@ -27,6 +27,15 @@ or to install the requirements manually:
 .. _`qpid-python`: https://pypi.org/project/qpid-python/
 .. _`qpid-tools`: https://pypi.org/project/qpid-tools/
 
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
 Authentication
 ==============
 
diff --git a/kombu/transport/redis.py b/kombu/transport/redis.py
index abd41785..e7edfd00 100644
--- a/kombu/transport/redis.py
+++ b/kombu/transport/redis.py
@@ -1,4 +1,53 @@
-"""Redis transport."""
+"""Redis transport module for Kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: No
+
+Connection String
+=================
+Connection string has the following format:
+
+.. code-block::
+
+    redis://REDIS_ADDRESS[:PORT][/VIRTUALHOST]
+    rediss://REDIS_ADDRESS[:PORT][/VIRTUALHOST]
+
+To use sentinel for dynamic Redis discovery,
+the connection string has following format:
+
+.. code-block::
+
+    sentinel://SENTINEL_ADDRESS[:PORT]
+
+Transport Options
+=================
+* ``sep``
+* ``ack_emulation``: (bool) If set to True transport will
+  simulate Acknowledge of AMQP protocol.
+* ``unacked_key``
+* ``unacked_index_key``
+* ``unacked_mutex_key``
+* ``unacked_mutex_expire``
+* ``visibility_timeout``
+* ``unacked_restore_limit``
+* ``fanout_prefix``
+* ``fanout_patterns``
+* ``socket_timeout``
+* ``socket_connect_timeout``
+* ``socket_keepalive``
+* ``socket_keepalive_options``
+* ``queue_order_strategy``
+* ``max_connections``
+* ``health_check_interval``
+* ``retry_on_timeout``
+* ``priority_steps``
+"""
 
 import numbers
 import socket
diff --git a/kombu/transport/sqlalchemy/__init__.py b/kombu/transport/sqlalchemy/__init__.py
index ccd51b0c..6ae04d0d 100644
--- a/kombu/transport/sqlalchemy/__init__.py
+++ b/kombu/transport/sqlalchemy/__init__.py
@@ -1,4 +1,55 @@
-"""Kombu transport using SQLAlchemy as the message store."""
+"""SQLAlchemy Transport module for kombu.
+
+Kombu transport using SQL Database as the message store.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: yes
+* Supports Topic: yes
+* Supports Fanout: no
+* Supports Priority: no
+* Supports TTL: no
+
+Connection String
+=================
+
+.. code-block::
+
+    sqla+SQL_ALCHEMY_CONNECTION_STRING
+    sqlalchemy+SQL_ALCHEMY_CONNECTION_STRING
+
+For details about ``SQL_ALCHEMY_CONNECTION_STRING`` see SQLAlchemy Engine Configuration documentation.
+Examples:
+
+.. code-block::
+
+    # PostgreSQL with default driver
+    sqla+postgresql://scott:tiger@localhost/mydatabase
+
+    # PostgreSQL with psycopg2 driver
+    sqla+postgresql+psycopg2://scott:tiger@localhost/mydatabase
+
+    # PostgreSQL with pg8000 driver
+    sqla+postgresql+pg8000://scott:tiger@localhost/mydatabase
+
+    # MySQL with default driver
+    sqla+mysql://scott:tiger@localhost/foo
+
+    # MySQL with mysqlclient driver (a maintained fork of MySQL-Python)
+    sqla+mysql+mysqldb://scott:tiger@localhost/foo
+
+    # MySQL with PyMySQL driver
+    sqla+mysql+pymysql://scott:tiger@localhost/foo
+
+Transport Options
+=================
+
+* ``queue_tablename``: Name of table storing queues.
+* ``message_tablename``: Name of table storing messages.
+
+Moreover parameters of :func:`sqlalchemy.create_engine()` function can be passed as transport options.
+"""
 # SQLAlchemy overrides != False to have special meaning and pep8 complains
 # flake8: noqa
 
diff --git a/kombu/transport/zookeeper.py b/kombu/transport/zookeeper.py
index 30b0e292..bfb557fe 100644
--- a/kombu/transport/zookeeper.py
+++ b/kombu/transport/zookeeper.py
@@ -1,25 +1,45 @@
-"""Zookeeper transport.
+# copyright: (c) 2010 - 2013 by Mahendra M.
+# license: BSD, see LICENSE for more details.
 
-:copyright: (c) 2010 - 2013 by Mahendra M.
-:license: BSD, see LICENSE for more details.
+"""Zookeeper transport module for kombu.
 
-**Synopsis**
-
-Connects to a zookeeper node as <server>:<port>/<vhost>
-The <vhost> becomes the base for all the other znodes.  So we can use
-it like a vhost.
-
-This uses the built-in kazoo recipe for queues
+Zookeeper based transport. This transport uses the built-in kazoo Zookeeper
+based queue implementation.
 
 **References**
 
-- https://zookeeper.apache.org/doc/trunk/recipes.html#sc_recipes_Queues
+- https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Queues
 - https://kazoo.readthedocs.io/en/latest/api/recipe/queue.html
 
 **Limitations**
 This queue does not offer reliable consumption.  An entry is removed from
 the queue prior to being processed.  So if an error occurs, the consumer
 has to re-queue the item or it will be lost.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: Yes
+* Supports TTL: No
+
+Connection String
+=================
+Connects to a zookeeper node as:
+
+.. code-block::
+
+    zookeeper://SERVER:PORT/VHOST
+
+The <vhost> becomes the base for all the other znodes.  So we can use
+it like a vhost.
+
+
+Transport Options
+=================
+
 """
 
 import os