Merge pull request #150 from tpazderka/MetadataStore

Metadata store

21 files changed, 651 insertions, 212 deletions

diff --git a/src/saml2/mdstore.py b/src/saml2/mdstore.py
index 361784f7..e2c6abe5 100644
--- a/src/saml2/mdstore.py
+++ b/src/saml2/mdstore.py
@@ -113,12 +113,211 @@ def repack_cert(cert):
 
 
 class MetaData(object):
-    def __init__(self, onts, attrc, metadata="", node_name=None,
-                 check_validity=True, security=None, **kwargs):
+    def __init__(self, onts, attrc, metadata='', node_name=None, check_validity=True,
+                 security=None, **kwargs):
         self.onts = onts
         self.attrc = attrc
-        self.entity = {}
         self.metadata = metadata
+
+    def items(self):
+        '''
+        Returns list of items contained in the storage
+        '''
+        raise NotImplementedError
+
+    def keys(self):
+        '''
+        Returns keys (identifiers) of items in storage
+        '''
+        raise NotImplementedError
+
+    def values(self):
+        '''
+        Returns values of items in storage
+        '''
+        raise NotImplementedError
+
+    def __len__(self):
+        '''
+        Returns number of stored items
+        '''
+        raise NotImplementedError
+
+    def __contains__(self, item):
+        '''
+        Returns True if the storage contains item
+        '''
+        raise NotImplementedError
+
+    def __getitem__(self, item):
+        '''
+        Returns the item specified by the key
+        '''
+        raise NotImplementedError
+
+    def __setitem__(self, key, value):
+        '''
+        Sets a key to a value
+        '''
+        raise NotImplementedError
+
+    def __delitem__(self, key):
+        '''
+        Removes key from storage
+        '''
+        raise NotImplementedError
+
+    def do_entity_descriptor(self, entity_descr):
+        '''
+        #FIXME - Add description
+        '''
+        raise NotImplementedError
+
+    def parse(self, xmlstr):
+        '''
+        #FIXME - Add description
+        '''
+        raise NotImplementedError
+
+    def load(self):
+        '''
+        Loads the metadata
+        '''
+        self.parse(self.metadata)
+
+    def service(self, entity_id, typ, service, binding=None):
+        """ Get me all services with a specified
+        entity ID and type, that supports the specified version of binding.
+
+        :param entity_id: The EntityId
+        :param typ: Type of service (idp, attribute_authority, ...)
+        :param service: which service that is sought for
+        :param binding: A binding identifier
+        :return: list of service descriptions.
+            Or if no binding was specified a list of 2-tuples (binding, srv)
+        """
+        raise NotImplementedError
+
+    def ext_service(self, entity_id, typ, service, binding):
+        try:
+            srvs = self[entity_id][typ]
+        except KeyError:
+            return None
+
+        if not srvs:
+            return srvs
+
+        res = []
+        for srv in srvs:
+            if "extensions" in srv:
+                for elem in srv["extensions"]["extension_elements"]:
+                    if elem["__class__"] == service:
+                        if elem["binding"] == binding:
+                            res.append(elem)
+
+        return res
+
+    def any(self, typ, service, binding=None):
+        """
+        Return any entity that matches the specification
+
+        :param typ:
+        :param service:
+        :param binding:
+        :return:
+        """
+        res = {}
+        for ent in self.keys():
+            bind = self.service(ent, typ, service, binding)
+            if bind:
+                res[ent] = bind
+
+        return res
+
+    def bindings(self, entity_id, typ, service):
+        """
+        Get me all the bindings that are registered for a service entity
+
+        :param entity_id:
+        :param service:
+        :return:
+        """
+        return self.service(entity_id, typ, service)
+
+    def attribute_requirement(self, entity_id, index=None):
+        """ Returns what attributes the SP requires and which are optional
+        if any such demands are registered in the Metadata.
+
+        :param entity_id: The entity id of the SP
+        :param index: which of the attribute consumer services its all about
+            if index=None then return all attributes expected by all
+            attribute_consuming_services.
+        :return: 2-tuple, list of required and list of optional attributes
+        """
+        raise NotImplementedError
+
+    def dumps(self):
+        return json.dumps(self.items(), indent=2)
+
+    def with_descriptor(self, descriptor):
+        '''
+        Returns any entities with the specified descriptor
+        '''
+        res = {}
+        desc = "%s_descriptor" % descriptor
+        for eid, ent in self.items():
+            if desc in ent:
+                res[eid] = ent
+        return res
+
+    def __str__(self):
+        return "%s" % self.items()
+
+    def construct_source_id(self):
+        raise NotImplementedError
+
+    def entity_categories(self, entity_id):
+        res = []
+        if "extensions" in self[entity_id]:
+            for elem in self[entity_id]["extensions"]["extension_elements"]:
+                if elem["__class__"] == ENTITYATTRIBUTES:
+                    for attr in elem["attribute"]:
+                        res.append(attr["text"])
+
+        return res
+
+    def __eq__(self, other):
+        try:
+            assert isinstance(other, MetaData)
+        except AssertionError:
+            return False
+
+        if len(self.entity) != len(other.entity):
+            return False
+
+        if set(self.entity.keys()) != set(other.entity.keys()):
+            return False
+
+        for key, item in self.entity.items():
+            try:
+                assert item == other[key]
+            except AssertionError:
+                return False
+
+        return True
+
+    def certs(self, entity_id, descriptor, use="signing"):
+        '''
+        Returns certificates for the given Entity
+        '''
+        raise NotImplementedError
+
+
+class InMemoryMetaData(MetaData):
+    def __init__(self, onts, attrc, metadata="", node_name=None,
+                 check_validity=True, security=None, **kwargs):
+        super(InMemoryMetaData, self).__init__(onts, attrc, metadata=metadata)
+        self.entity = {}
         self.security = security
         self.node_name = node_name
         self.entities_descr = None
@@ -221,9 +420,6 @@ class MetaData(object):
             for entity_descr in self.entities_descr.entity_descriptor:
                 self.do_entity_descriptor(entity_descr)
 
-    def load(self):
-        self.parse(self.metadata)
-
     def service(self, entity_id, typ, service, binding=None):
         """ Get me all services with a specified
         entity ID and type, that supports the specified version of binding.
@@ -235,7 +431,6 @@ class MetaData(object):
         :return: list of service descriptions.
             Or if no binding was specified a list of 2-tuples (binding, srv)
         """
-
         logger.debug("service(%s, %s, %s, %s)" % (entity_id, typ, service,
                                                   binding))
         try:
@@ -266,53 +461,6 @@ class MetaData(object):
         logger.debug("service => %s" % res)
         return res
 
-    def ext_service(self, entity_id, typ, service, binding):
-        try:
-            srvs = self[entity_id][typ]
-        except KeyError:
-            return None
-
-        if not srvs:
-            return srvs
-
-        res = []
-        for srv in srvs:
-            if "extensions" in srv:
-                for elem in srv["extensions"]["extension_elements"]:
-                    if elem["__class__"] == service:
-                        if elem["binding"] == binding:
-                            res.append(elem)
-
-        return res
-
-    def any(self, typ, service, binding=None):
-        """
-        Return any entity that matches the specification
-
-        :param typ:
-        :param service:
-        :param binding:
-        :return:
-        """
-        res = {}
-        for ent in self.keys():
-            bind = self.service(ent, typ, service, binding)
-            if bind:
-                res[ent] = bind
-
-        return res
-
-    def bindings(self, entity_id, typ, service):
-        """
-        Get me all the bindings that are registered for a service entity
-
-        :param entity_id:
-        :param service:
-        :return:
-        """
-
-        return self.service(entity_id, typ, service)
-
     def attribute_requirement(self, entity_id, index=None):
         """ Returns what attributes the SP requires and which are optional
         if any such demands are registered in the Metadata.
@@ -323,7 +471,6 @@ class MetaData(object):
             attribute_consuming_services.
         :return: 2-tuple, list of required and list of optional attributes
         """
-
         res = {"required": [], "optional": []}
 
         try:
@@ -336,20 +483,6 @@ class MetaData(object):
 
         return res
 
-    def dumps(self):
-        return json.dumps(self.items(), indent=2)
-
-    def with_descriptor(self, descriptor):
-        res = {}
-        desc = "%s_descriptor" % descriptor
-        for eid, ent in self.items():
-            if desc in ent:
-                res[eid] = ent
-        return res
-
-    def __str__(self):
-        return "%s" % self.items()
-
     def construct_source_id(self):
         res = {}
         for eid, ent in self.items():
@@ -364,36 +497,6 @@ class MetaData(object):
 
         return res
 
-    def entity_categories(self, entity_id):
-        res = []
-        if "extensions" in self[entity_id]:
-            for elem in self[entity_id]["extensions"]["extension_elements"]:
-                if elem["__class__"] == ENTITYATTRIBUTES:
-                    for attr in elem["attribute"]:
-                        res.append(attr["text"])
-
-        return res
-
-    def __eq__(self, other):
-        try:
-            assert isinstance(other, MetaData)
-        except AssertionError:
-            return False
-
-        if len(self.entity) != len(other.entity):
-            return False
-
-        if set(self.entity.keys()) != set(other.entity.keys()):
-            return False
-
-        for key, item in self.entity.items():
-            try:
-                assert item == other[key]
-            except AssertionError:
-                return False
-
-        return True
-
     def certs(self, entity_id, descriptor, use="signing"):
         ent = self.__getitem__(entity_id)
         if descriptor == "any":
@@ -434,13 +537,15 @@ class MetaData(object):
         return res
 
 
-class MetaDataFile(MetaData):
+class MetaDataFile(InMemoryMetaData):
     """
     Handles Metadata file on the same machine. The format of the file is
     the SAML Metadata format.
     """
-    def __init__(self, onts, attrc, filename, cert=None, **kwargs):
-        MetaData.__init__(self, onts, attrc, **kwargs)
+    def __init__(self, onts, attrc, filename=None, cert=None, **kwargs):
+        super(MetaDataFile, self).__init__(onts, attrc, **kwargs)
+        if not file:
+            raise SAMLError('No file specified.')
         self.filename = filename
         self.cert = cert
 
@@ -471,7 +576,7 @@ class MetaDataLoader(MetaDataFile):
     """
     def __init__(self, onts, attrc, loader_callable, cert=None,
                  security=None, **kwargs):
-        MetaData.__init__(self, onts, attrc, **kwargs)
+        super(MetaDataLoader, self).__init__(onts, attrc, **kwargs)
         self.metadata_provider_callable = self.get_metadata_loader(
             loader_callable)
         self.cert = cert
@@ -507,25 +612,32 @@ class MetaDataLoader(MetaDataFile):
         return self.metadata_provider_callable()
 
 
-class MetaDataExtern(MetaData):
+class MetaDataExtern(InMemoryMetaData):
     """
     Class that handles metadata store somewhere on the net.
     Accessible but HTTP GET.
     """
 
-    def __init__(self, onts, attrc, url, security, cert, http, **kwargs):
+    def __init__(self, onts, attrc, url=None, security=None, cert=None, http=None, **kwargs):
         """
         :params onts:
         :params attrc:
-        :params url:
+        :params url: Location of the metadata
         :params security: SecurityContext()
-        :params cert:
+        :params cert: CertificMDloaderate used to sign the metadata
         :params http:
         """
-        MetaData.__init__(self, onts, attrc, **kwargs)
-        self.url = url
+        super(MetaDataExtern, self).__init__(onts, attrc, **kwargs)
+        if not url:
+            raise SAMLError('URL not specified.')
+        else:
+            self.url = url
+        if not cert:
+            raise SAMLError('No certificate specified.')
+        else:
+            self.cert = cert
+
         self.security = security
-        self.cert = cert
         self.http = http
 
     def load(self):
@@ -554,13 +666,13 @@ class MetaDataExtern(MetaData):
         return False
 
 
-class MetaDataMD(MetaData):
+class MetaDataMD(InMemoryMetaData):
     """
     Handles locally stored metadata, the file format is the text representation
     of the Python representation of the metadata.
     """
     def __init__(self, onts, attrc, filename, **kwargs):
-        MetaData.__init__(self, onts, attrc, **kwargs)
+        super(MetaDataMD, self).__init__(onts, attrc, **kwargs)
         self.filename = filename
 
     def load(self):
@@ -571,7 +683,7 @@ class MetaDataMD(MetaData):
 SAML_METADATA_CONTENT_TYPE = 'application/samlmetadata+xml'
 
 
-class MetaDataMDX(MetaData):
+class MetaDataMDX(InMemoryMetaData):
     """ Uses the md protocol to fetch entity information
     """
     def __init__(self, entity_transform, onts, attrc, url, security, cert,
@@ -587,7 +699,7 @@ class MetaDataMDX(MetaData):
         :params cert:
         :params http:
         """
-        MetaData.__init__(self, onts, attrc, **kwargs)
+        super(MetaDataMDX, self).__init__(onts, attrc, **kwargs)
         self.url = url
         self.security = security
         self.cert = cert
@@ -676,7 +788,6 @@ class MetadataStore(object):
                     _args[_key] = kwargs[_key]
                 except KeyError:
                     pass
-
             _md = MetaDataExtern(self.onts, self.attrc,
                                  kwargs["url"], self.security,
                                  kwargs["cert"], self.http, **_args)
@@ -688,19 +799,52 @@ class MetadataStore(object):
             _md = MetaDataLoader(self.onts, self.attrc, args[0])
         else:
             raise SAMLError("Unknown metadata type '%s'" % typ)
-
         _md.load()
         self.metadata[key] = _md
 
     def imp(self, spec):
-        for key, vals in spec.items():
-            for val in vals:
-                if isinstance(val, dict):
-                    if not self.check_validity:
-                        val["check_validity"] = False
-                    self.load(key, **val)
-                else:
-                    self.load(key, val)
+        # This serves as a backwards compatibility
+        if type(spec) is dict:
+            # Old style...
+            for key, vals in spec.items():
+                for val in vals:
+                    if isinstance(val, dict):
+                        if not self.check_validity:
+                            val["check_validity"] = False
+                        self.load(key, **val)
+                    else:
+                        self.load(key, val)
+        else:
+            for item in spec:
+                try:
+                    key = item['class']
+                except (KeyError, AttributeError):
+                    raise SAMLError("Misconfiguration in metadata %s" % item)
+                mod, clas = key.rsplit('.', 1)
+                try:
+                    mod = import_module(mod)
+                    MDloader = getattr(mod, clas)
+                except (ImportError, AttributeError):
+                    raise SAMLError("Unknown metadata loader %s" % key)
+
+                # Separately handle MDExtern
+                if MDloader == MetaDataExtern:
+                    item['http'] = self.http
+                    item['security'] = self.security
+
+                for key in item['metadata']:
+                    # Separately handle MetaDataFile and directory
+                    if MDloader == MetaDataFile and os.path.isdir(key[0]):
+                        files = [f for f in listdir(key[0]) if isfile(join(key[0], f))]
+                        for fil in files:
+                            _fil = join(key[0], fil)
+                            _md = MetaDataFile(self.onts, self.attrc, _fil)
+                            _md.load()
+                            self.metadata[_fil] = _md
+                        return
+                    _md = MDloader(self.onts, self.attrc, *key)
+                    _md.load()
+                    self.metadata[key[0]] = _md
 
     def service(self, entity_id, typ, service, binding=None):
         known_entity = False
diff --git a/src/saml2/mongo_store.py b/src/saml2/mongo_store.py
index 30d8daf3..2dccdfb5 100644
--- a/src/saml2/mongo_store.py
+++ b/src/saml2/mongo_store.py
@@ -6,7 +6,7 @@ from pymongo.mongo_replica_set_client import MongoReplicaSetClient
 import pymongo.uri_parser
 import pymongo.errors
 from saml2.eptid import Eptid
-from saml2.mdstore import MetaData
+from saml2.mdstore import InMemoryMetaData
 from saml2.s_utils import PolicyError
 
 from saml2.ident import code, IdentDB, Unknown
@@ -377,9 +377,9 @@ def export_mdstore_to_mongo_db(mds, database, collection, sub_collection=""):
         mdb.store(key, **kwargs)
 
 
-class MetadataMDB(MetaData):
+class MetadataMDB(InMemoryMetaData):
     def __init__(self, onts, attrc, database="", collection=""):
-        MetaData.__init__(self, onts, attrc)
+        super(MetadataMDB, self).__init__(onts, attrc)
         self.mdb = MDB(database, collection)
         self.mdb.primary_key = "entity_id"
 
diff --git a/tests/disco_conf.py b/tests/disco_conf.py
index 253dc22e..68cf0388 100644
--- a/tests/disco_conf.py
+++ b/tests/disco_conf.py
@@ -19,7 +19,8 @@ CONFIG = {
     },
     "debug": 1,
     "xmlsec_binary": xmlsec_path,
-    "metadata": {
-        "local": [full_path("servera.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("servera.xml"), )],
+    }],
 }
diff --git a/tests/idp_all_conf.py b/tests/idp_all_conf.py
index 3ff2fa99..e4b0b7ee 100644
--- a/tests/idp_all_conf.py
+++ b/tests/idp_all_conf.py
@@ -87,10 +87,11 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     "xmlsec_path": ["/usr/local/bin", "/opt/local/bin"],
-    "metadata": {
-        "local": [full_path("servera.xml"),
-                  full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("servera.xml"), ),
+                     (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir": full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_conf.py b/tests/idp_conf.py
index f0d01804..ffac5589 100644
--- a/tests/idp_conf.py
+++ b/tests/idp_conf.py
@@ -45,10 +45,11 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     "xmlsec_binary": xmlsec_path,
-    "metadata": {
-        "local": [full_path("metadata_sp_1.xml"),
-                  full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata_sp_1.xml"), ),
+                     (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir": full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_conf_ec.py b/tests/idp_conf_ec.py
index 39b343cf..a2a670b2 100644
--- a/tests/idp_conf_ec.py
+++ b/tests/idp_conf_ec.py
@@ -38,10 +38,11 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     "xmlsec_binary": xmlsec_path,
-    "metadata": {
-        "local": [full_path("metadata_sp_1.xml"),
-                  full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata_sp_1.xml"), ),
+                  (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir": full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_conf_mdb.py b/tests/idp_conf_mdb.py
index f4afe1ca..1b8bc8be 100644
--- a/tests/idp_conf_mdb.py
+++ b/tests/idp_conf_mdb.py
@@ -88,10 +88,11 @@ CONFIG = {
     "cert_file": full_path("test.pem"),
     #"xmlsec_binary": None,
     "xmlsec_path": ["/opt/local/bin", "usr/local/bin"],
-    "metadata": {
-        "local": [full_path("servera.xml"),
-                  full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("servera.xml"), ),
+                  (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir": full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_slo_redirect_conf.py b/tests/idp_slo_redirect_conf.py
index 530f0fe0..8e5c48a8 100644
--- a/tests/idp_slo_redirect_conf.py
+++ b/tests/idp_slo_redirect_conf.py
@@ -33,9 +33,10 @@ CONFIG = {
     "key_file" : full_path("test.key"),
     "cert_file" : full_path("test.pem"),
     "xmlsec_binary" : None,
-    "metadata": {
-        "local": [full_path("sp_slo_redirect.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("sp_slo_redirect.xml"), )],
+    }],
     "attribute_map_dir" : full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_soap_conf.py b/tests/idp_soap_conf.py
index 110bccdf..e65c1b81 100644
--- a/tests/idp_soap_conf.py
+++ b/tests/idp_soap_conf.py
@@ -37,9 +37,10 @@ CONFIG={
     "key_file" : full_path("test.key"),
     "cert_file" : full_path("test.pem"),
     #"xmlsec_binary" : None,
-    "metadata": {
-        "local": [full_path("metadata.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir" : full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/idp_sp_conf.py b/tests/idp_sp_conf.py
index 2f7a7b50..c686cffe 100644
--- a/tests/idp_sp_conf.py
+++ b/tests/idp_sp_conf.py
@@ -50,9 +50,10 @@ CONFIG = {
     "key_file" : full_path("test.key"),
     "cert_file" : full_path("test.pem"),
     "xmlsec_binary" : None,
-    "metadata": {
-        "local": [full_path("metadata.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "attribute_map_dir" : full_path("attributemaps"),
     "organization": {
         "name": "Exempel AB",
diff --git a/tests/restrictive_idp_conf.py b/tests/restrictive_idp_conf.py
index 468c9782..ca089daa 100644
--- a/tests/restrictive_idp_conf.py
+++ b/tests/restrictive_idp_conf.py
@@ -37,8 +37,9 @@ CONFIG = {
     "key_file" : full_path("test.key"),
     "cert_file" : full_path("test.pem"),
     "xmlsec_binary" : None,
-    "metadata": {
-        "local": [full_path("sp_0.metadata")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("sp_0.metadata"), )],
+    }],
     "attribute_map_dir" : full_path("attributemaps"),
 }
diff --git a/tests/server_conf.py b/tests/server_conf.py
index 2ff8f6dc..bb9de901 100644
--- a/tests/server_conf.py
+++ b/tests/server_conf.py
@@ -21,9 +21,10 @@ CONFIG = {
     "cert_file": full_path("test.pem"),
     "ca_certs": full_path("cacerts.txt"),
     "xmlsec_binary": xmlsec_path,
-    "metadata": {
-        "local": [full_path("idp.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("idp.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "virtual_organization": {
         "urn:mace:example.com:it:tek": {
             "nameid_format": "urn:oid:1.3.6.1.4.1.1466.115.121.1.15-NameID",
diff --git a/tests/server_conf_syslog.py b/tests/server_conf_syslog.py
index 4dc6a85b..ce6d841f 100644
--- a/tests/server_conf_syslog.py
+++ b/tests/server_conf_syslog.py
@@ -22,9 +22,10 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     # "xmlsec_binary" : None,
-    "metadata": {
-        "local": [full_path("idp.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("idp.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "virtual_organization": {
         "urn:mace:example.com:it:tek": {
             "nameid_format": "urn:oid:1.3.6.1.4.1.1466.115.121.1.15-NameID",
diff --git a/tests/servera_conf.py b/tests/servera_conf.py
index 8d1c58c0..3ab8741b 100644
--- a/tests/servera_conf.py
+++ b/tests/servera_conf.py
@@ -51,9 +51,10 @@ CONFIG = {
     "cert_file": full_path("test.pem"),
     "ca_certs": full_path("cacerts.txt"),
     "xmlsec_binary": xmlsec_path,
-    "metadata": {
-        "local": [full_path("idp_all.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("idp_all.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "virtual_organization": {
         "urn:mace:example.com:it:tek": {
             "nameid_format": "urn:oid:1.3.6.1.4.1.1466.115.121.1.15-NameID",
diff --git a/tests/sp_1_conf.py b/tests/sp_1_conf.py
index 90bf0752..834ff15a 100644
--- a/tests/sp_1_conf.py
+++ b/tests/sp_1_conf.py
@@ -20,9 +20,10 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     "xmlsec_binary": None,
-    "metadata": {
-        "local": [full_path("idp.xml"), full_path("vo_metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("idp.xml"), ), (full_path("vo_metadata.xml"), )],
+    }],
     "virtual_organization": {
         "urn:mace:example.com:it:tek": {
             "nameid_format": "urn:oid:1.3.6.1.4.1.1466.115.121.1.15-NameID",
diff --git a/tests/sp_slo_redirect_conf.py b/tests/sp_slo_redirect_conf.py
index 104d2f74..7bc17a1b 100644
--- a/tests/sp_slo_redirect_conf.py
+++ b/tests/sp_slo_redirect_conf.py
@@ -27,9 +27,10 @@ CONFIG = {
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
     "xmlsec_binary": None,
-    "metadata": {
-        "local": [full_path("idp_slo_redirect.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("idp_slo_redirect.xml"), )],
+    }],
     "virtual_organization": {
         "urn:mace:example.com:it:tek": {
             "nameid_format": "urn:oid:1.3.6.1.4.1.1466.115.121.1.15-NameID",
diff --git a/tests/test_30_mdstore.py b/tests/test_30_mdstore.py
index 76a4a6ba..8e02f288 100644
--- a/tests/test_30_mdstore.py
+++ b/tests/test_30_mdstore.py
@@ -49,34 +49,44 @@ ONTS = {
 ATTRCONV = ac_factory(full_path("attributemaps"))
 
 METADATACONF = {
-    "1": {
-        "local": [full_path("swamid-1.0.xml")]
-    },
-    "2": {
-        "local": [full_path("InCommon-metadata.xml")]
-    },
-    "3": {
-        "local": [full_path("extended.xml")]
-    },
-    "7": {
-        "local": [full_path("metadata_sp_1.xml"),
-                  full_path("InCommon-metadata.xml")],
-        "remote": [
-            {"url": "https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2",
-             "cert": full_path("kalmar2.pem")}]
-    },
-    "4": {
-        "local": [full_path("metadata_example.xml")]
-    },
-    "5": {
-        "local": [full_path("metadata.aaitest.xml")]
-    },
-    "8": {
-        "mdfile": [full_path("swamid.md")]
-    },
-    "9": {
-        "local": [full_path("metadata")]
-    }
+    "1": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("swamid-1.0.xml"), )],
+    }],
+    "2": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("InCommon-metadata.xml"), )],
+    }],
+    "3": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("extended.xml"), )],
+    }],
+    "7": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata_sp_1.xml"), ),
+                     (full_path("InCommon-metadata.xml"), )], },
+          {
+        "class": "saml2.mdstore.MetaDataExtern",
+        "metadata": [
+            ("https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2",
+             full_path("kalmar2.pem")), ],
+    }],
+    "4": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata_example.xml"), )],
+    }],
+    "5": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata.aaitest.xml"), )],
+    }],
+    "8": [{
+        "class": "saml2.mdstore.MetaDataMD",
+        "metadata": [(full_path("swamid.md"), )],
+    }],
+    "9": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata"), )]
+    }]
 }
 
 
@@ -117,13 +127,13 @@ def test_swami_1():
     lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
     assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
                        'eduPersonScopedAffiliation'])
-                
+
     wants = mds.attribute_requirement('https://beta.lobber.se/shibboleth')
     assert wants["required"] == []
     lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
     assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
                        'eduPersonScopedAffiliation', 'eduPersonEntitlement'])
-                
+
 
 def test_incommon_1():
     mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
diff --git a/tests/test_30_mdstore_old.py b/tests/test_30_mdstore_old.py
new file mode 100644
index 00000000..0f3d3d04
--- /dev/null
+++ b/tests/test_30_mdstore_old.py
@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import datetime
+import re
+from urllib import quote_plus
+from saml2.httpbase import HTTPBase
+
+from saml2.mdstore import MetadataStore, MetaDataMDX
+from saml2.mdstore import destinations
+from saml2.mdstore import name
+
+from saml2 import md
+from saml2 import sigver
+from saml2 import BINDING_SOAP
+from saml2 import BINDING_HTTP_REDIRECT
+from saml2 import BINDING_HTTP_POST
+from saml2 import BINDING_HTTP_ARTIFACT
+from saml2 import saml
+from saml2 import config
+from saml2.attribute_converter import ac_factory
+from saml2.attribute_converter import d_to_local_name
+
+from saml2.extension import mdui
+from saml2.extension import idpdisc
+from saml2.extension import dri
+from saml2.extension import mdattr
+from saml2.extension import ui
+from saml2.s_utils import UnknownPrincipal
+import xmldsig
+import xmlenc
+
+from pathutils import full_path
+
+sec_config = config.Config()
+#sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
+
+ONTS = {
+    saml.NAMESPACE: saml,
+    mdui.NAMESPACE: mdui,
+    mdattr.NAMESPACE: mdattr,
+    dri.NAMESPACE: dri,
+    ui.NAMESPACE: ui,
+    idpdisc.NAMESPACE: idpdisc,
+    md.NAMESPACE: md,
+    xmldsig.NAMESPACE: xmldsig,
+    xmlenc.NAMESPACE: xmlenc
+}
+
+ATTRCONV = ac_factory(full_path("attributemaps"))
+
+METADATACONF = {
+    "1": {
+        "local": [full_path("swamid-1.0.xml")]
+    },
+    "2": {
+        "local": [full_path("InCommon-metadata.xml")]
+    },
+    "3": {
+        "local": [full_path("extended.xml")]
+    },
+    "7": {
+        "local": [full_path("metadata_sp_1.xml"),
+                  full_path("InCommon-metadata.xml")],
+        "remote": [
+            {"url": "https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2",
+             "cert": full_path("kalmar2.pem")}]
+    },
+    "4": {
+        "local": [full_path("metadata_example.xml")]
+    },
+    "5": {
+        "local": [full_path("metadata.aaitest.xml")]
+    },
+    "8": {
+        "mdfile": [full_path("swamid.md")]
+    },
+    "9": {
+        "local": [full_path("metadata")]
+    }
+}
+
+
+def _eq(l1, l2):
+    return set(l1) == set(l2)
+
+
+def _fix_valid_until(xmlstring):
+    new_date = datetime.datetime.now() + datetime.timedelta(days=1)
+    new_date = new_date.strftime("%Y-%m-%dT%H:%M:%SZ")
+    return re.sub(r' validUntil=".*?"', ' validUntil="%s"' % new_date,
+                  xmlstring)
+
+
+def test_swami_1():
+    UMU_IDP = 'https://idp.umu.se/saml2/idp/metadata.php'
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["1"])
+    assert len(mds) == 1  # One source
+    idps = mds.with_descriptor("idpsso")
+    assert idps.keys()
+    idpsso = mds.single_sign_on_service(UMU_IDP)
+    assert len(idpsso) == 1
+    assert destinations(idpsso) == [
+        'https://idp.umu.se/saml2/idp/SSOService.php']
+
+    _name = name(mds[UMU_IDP])
+    assert _name == u'Umeå University (SAML2)'
+    certs = mds.certs(UMU_IDP, "idpsso", "signing")
+    assert len(certs) == 1
+
+    sps = mds.with_descriptor("spsso")
+    assert len(sps) == 108
+
+    wants = mds.attribute_requirement('https://connect8.sunet.se/shibboleth')
+    lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
+    assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
+                       'eduPersonScopedAffiliation'])
+
+    wants = mds.attribute_requirement('https://beta.lobber.se/shibboleth')
+    assert wants["required"] == []
+    lnamn = [d_to_local_name(mds.attrc, attr) for attr in wants["optional"]]
+    assert _eq(lnamn, ['eduPersonPrincipalName', 'mail', 'givenName', 'sn',
+                       'eduPersonScopedAffiliation', 'eduPersonEntitlement'])
+
+
+def test_incommon_1():
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["2"])
+
+    print mds.entities()
+    assert mds.entities() > 1700
+    idps = mds.with_descriptor("idpsso")
+    print idps.keys()
+    assert len(idps) > 300  # ~ 18%
+    try:
+        _ = mds.single_sign_on_service('urn:mace:incommon:uiuc.edu')
+    except UnknownPrincipal:
+        pass
+
+    idpsso = mds.single_sign_on_service('urn:mace:incommon:alaska.edu')
+    assert len(idpsso) == 1
+    print idpsso
+    assert destinations(idpsso) == [
+        'https://idp.alaska.edu/idp/profile/SAML2/Redirect/SSO']
+
+    sps = mds.with_descriptor("spsso")
+
+    acs_sp = []
+    for nam, desc in sps.items():
+        if "attribute_consuming_service" in desc:
+            acs_sp.append(nam)
+
+    assert len(acs_sp) == 0
+
+    # Look for attribute authorities
+    aas = mds.with_descriptor("attribute_authority")
+
+    print aas.keys()
+    assert len(aas) == 180
+
+
+def test_ext_2():
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["3"])
+    # No specific binding defined
+
+    ents = mds.with_descriptor("spsso")
+    for binding in [BINDING_SOAP, BINDING_HTTP_POST, BINDING_HTTP_ARTIFACT,
+                    BINDING_HTTP_REDIRECT]:
+        assert mds.single_logout_service(ents.keys()[0], binding, "spsso")
+
+
+def test_example():
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["4"])
+    assert len(mds.keys()) == 1
+    idps = mds.with_descriptor("idpsso")
+
+    assert idps.keys() == [
+        'http://xenosmilus.umdc.umu.se/simplesaml/saml2/idp/metadata.php']
+    certs = mds.certs(
+        'http://xenosmilus.umdc.umu.se/simplesaml/saml2/idp/metadata.php',
+        "idpsso", "signing")
+    assert len(certs) == 1
+
+
+def test_switch_1():
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["5"])
+    assert len(mds.keys()) > 160
+    idps = mds.with_descriptor("idpsso")
+    print idps.keys()
+    idpsso = mds.single_sign_on_service(
+        'https://aai-demo-idp.switch.ch/idp/shibboleth')
+    assert len(idpsso) == 1
+    print idpsso
+    assert destinations(idpsso) == [
+        'https://aai-demo-idp.switch.ch/idp/profile/SAML2/Redirect/SSO']
+    assert len(idps) > 30
+    aas = mds.with_descriptor("attribute_authority")
+    print aas.keys()
+    aad = aas['https://aai-demo-idp.switch.ch/idp/shibboleth']
+    print aad.keys()
+    assert len(aad["attribute_authority_descriptor"]) == 1
+    assert len(aad["idpsso_descriptor"]) == 1
+
+    sps = mds.with_descriptor("spsso")
+    dual = [eid for eid, ent in idps.items() if eid in sps]
+    print len(dual)
+    assert len(dual) == 0
+
+
+def test_metadata_file():
+    sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["8"])
+    print len(mds.keys())
+    assert len(mds.keys()) == 560
+
+
+def test_mdx_service():
+    sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
+    http = HTTPBase(verify=False, ca_bundle=None)
+
+    mdx = MetaDataMDX(quote_plus, ONTS.values(), ATTRCONV,
+                      "http://pyff-test.nordu.net",
+                      sec_config, None, http)
+    foo = mdx.service("https://idp.umu.se/saml2/idp/metadata.php",
+                      "idpsso_descriptor", "single_sign_on_service")
+
+    assert len(foo) == 1
+    assert foo.keys()[0] == BINDING_HTTP_REDIRECT
+
+
+def test_mdx_certs():
+    sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
+    http = HTTPBase(verify=False, ca_bundle=None)
+
+    mdx = MetaDataMDX(quote_plus, ONTS.values(), ATTRCONV,
+                      "http://pyff-test.nordu.net",
+                      sec_config, None, http)
+    foo = mdx.certs("https://idp.umu.se/saml2/idp/metadata.php", "idpsso")
+
+    assert len(foo) == 1
+
+
+def test_load_local_dir():
+    sec_config.xmlsec_binary = sigver.get_xmlsec_binary(["/opt/local/bin"])
+    mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
+                        disable_ssl_certificate_validation=True)
+
+    mds.imp(METADATACONF["9"])
+    print mds
+    assert len(mds) == 3  # Three sources
+    assert len(mds.keys()) == 4  # number of idps
+
+if __name__ == "__main__":
+    test_mdx_certs()
diff --git a/tests/test_31_config.py b/tests/test_31_config.py
index 245714b3..ea25abbb 100644
--- a/tests/test_31_config.py
+++ b/tests/test_31_config.py
@@ -32,10 +32,11 @@ sp1 = {
     },
     "key_file": full_path("test.key"),
     "cert_file": full_path("test.pem"),
-    "metadata": {
-        "local": [full_path("metadata.xml"),
-                  full_path("urn-mace-swami.se-swamid-test-1.0-metadata.xml")],
-    },
+    "metadata": [{
+        "class": "saml2.mdstore.MetaDataFile",
+        "metadata": [(full_path("metadata.xml"), ),
+                  (full_path("urn-mace-swami.se-swamid-test-1.0-metadata.xml"), )],
+    }],
     "virtual_organization": {
         "coip": {
             "nameid_format": "urn:oasis:names:tc:SAML:2.0:nameid-format:transient",
diff --git a/tests/test_37_entity_categories.py b/tests/test_37_entity_categories.py
index 63fdefa4..3e95458f 100644
--- a/tests/test_37_entity_categories.py
+++ b/tests/test_37_entity_categories.py
@@ -35,7 +35,7 @@ __author__ = 'rolandh'
 
 MDS = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
                     disable_ssl_certificate_validation=True)
-MDS.imp({"mdfile": [full_path("swamid.md")]})
+MDS.imp([{"class": "saml2.mdstore.MetaDataMD", "metadata": [(full_path("swamid.md"), )]}])
 
 
 def _eq(l1, l2):
@@ -91,7 +91,7 @@ def test_filter_ava3():
 
     mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
                         disable_ssl_certificate_validation=True)
-    mds.imp({"local": [full_path("entity_cat_sfs_hei.xml")]})
+    mds.imp([{"class": "saml2.mdstore.MetaDataFile", "metadata": [(full_path("entity_cat_sfs_hei.xml"), )]}])
 
     ava = {"givenName": ["Derek"], "sn": ["Jeter"],
            "mail": ["derek@nyy.mlb.com"], "c": ["USA"],
@@ -114,7 +114,7 @@ def test_filter_ava4():
 
     mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
                         disable_ssl_certificate_validation=True)
-    mds.imp({"local": [full_path("entity_cat_re_nren.xml")]})
+    mds.imp([{"class": "saml2.mdstore.MetaDataFile", "metadata": [(full_path("entity_cat_re_nren.xml"), )]}])
 
     ava = {"givenName": ["Derek"], "sn": ["Jeter"],
            "mail": ["derek@nyy.mlb.com"], "c": ["USA"],
@@ -138,7 +138,7 @@ def test_filter_ava5():
 
     mds = MetadataStore(ONTS.values(), ATTRCONV, sec_config,
                         disable_ssl_certificate_validation=True)
-    mds.imp({"local": [full_path("entity_cat_re.xml")]})
+    mds.imp([{"class": "saml2.mdstore.MetaDataFile", "metadata": [(full_path("entity_cat_re.xml"), )]}])
 
     ava = {"givenName": ["Derek"], "sn": ["Jeter"],
            "mail": ["derek@nyy.mlb.com"], "c": ["USA"],
diff --git a/tests/test_76_metadata_in_mdb.py b/tests/test_76_metadata_in_mdb.py
index e6b6f0b1..6520ec1e 100644
--- a/tests/test_76_metadata_in_mdb.py
+++ b/tests/test_76_metadata_in_mdb.py
@@ -53,7 +53,7 @@ def test_metadata():
                         disable_ssl_certificate_validation=True)
 
     # Import metadata from local file.
-    mds.imp({"local": [full_path("swamid-2.0.xml")]})
+    mds.imp([{"class": "saml2.mdstore.MetaDataFile", "metadata": [(full_path("swamid-2.0.xml"), )]}])
     assert len(mds) == 1  # One source
 
     try: