Parse nested files if they are template

Parse nested files if they are in template format.

Closes-Bug: #1429036
Change-Id: I6b01ed853edac815510e89fbfaf775084a9c5ac7

2 files changed, 141 insertions, 27 deletions

diff --git a/heatclient/common/template_utils.py b/heatclient/common/template_utils.py
index 3c05687..cdf92b2 100644
--- a/heatclient/common/template_utils.py
+++ b/heatclient/common/template_utils.py
@@ -66,17 +66,20 @@ def get_template_contents(template_file=None, template_url=None,
     if files is None:
         files = {}
     resolve_template_get_files(template, files, tmpl_base_url)
-    resolve_template_type(template, files, tmpl_base_url)
     return files, template
 
 
 def resolve_template_get_files(template, files, template_base_url):
 
     def ignore_if(key, value):
-        if key != 'get_file':
+        if key != 'get_file' and key != 'type':
             return True
         if not isinstance(value, six.string_types):
             return True
+        if (key == 'type' and
+                not value.endswith(('.yaml', '.template'))):
+            return True
+        return False
 
     def recurse_if(value):
         return isinstance(value, (dict, list))
@@ -85,26 +88,18 @@ def resolve_template_get_files(template, files, template_base_url):
                       ignore_if, recurse_if)
 
 
-def resolve_template_type(template, files, template_base_url):
-
-    def ignore_if(key, value):
-        if key != 'type':
-            return True
-        if not isinstance(value, six.string_types):
-            return True
-        if not value.endswith(('.yaml', '.template')):
-            return True
+def is_template(file_content):
+    try:
+        if isinstance(file_content, six.binary_type):
+            file_content = file_content.decode('utf-8')
+        template_format.parse(file_content)
+    except (ValueError, TypeError):
         return False
-
-    def recurse_if(value):
-        return isinstance(value, (dict, list))
-
-    get_file_contents(template, files, template_base_url,
-                      ignore_if, recurse_if, file_is_template=True)
+    return True
 
 
 def get_file_contents(from_data, files, base_url=None,
-                      ignore_if=None, recurse_if=None, file_is_template=False):
+                      ignore_if=None, recurse_if=None):
 
     if recurse_if and recurse_if(from_data):
         if isinstance(from_data, dict):
@@ -112,8 +107,7 @@ def get_file_contents(from_data, files, base_url=None,
         else:
             recurse_data = from_data
         for value in recurse_data:
-            get_file_contents(value, files, base_url, ignore_if, recurse_if,
-                              file_is_template=file_is_template)
+            get_file_contents(value, files, base_url, ignore_if, recurse_if)
 
     if isinstance(from_data, dict):
         for key, value in iter(from_data.items()):
@@ -125,12 +119,11 @@ def get_file_contents(from_data, files, base_url=None,
 
             str_url = parse.urljoin(base_url, value)
             if str_url not in files:
-                if file_is_template:
+                file_content = read_url_content(str_url)
+                if is_template(file_content):
                     template = get_template_contents(
                         template_url=str_url, files=files)[1]
                     file_content = jsonutils.dumps(template)
-                else:
-                    file_content = utils.read_url_content(str_url)
                 files[str_url] = file_content
             # replace the data value with the normalised absolute URL
             from_data[key] = str_url
@@ -219,9 +212,9 @@ def resolve_environment_urls(resource_registry, files, env_base_url):
             # don't need downloading.
             return True
 
-    get_file_contents(rr, files, base_url, ignore_if, file_is_template=True)
+    get_file_contents(rr, files, base_url, ignore_if)
 
     for res_name, res_dict in iter(rr.get('resources', {}).items()):
         res_base_url = res_dict.get('base_url', base_url)
         get_file_contents(
-            res_dict, files, res_base_url, ignore_if, file_is_template=True)
+            res_dict, files, res_base_url, ignore_if)
diff --git a/heatclient/tests/test_template_utils.py b/heatclient/tests/test_template_utils.py
index f486154..dbd0b8f 100644
--- a/heatclient/tests/test_template_utils.py
+++ b/heatclient/tests/test_template_utils.py
@@ -44,6 +44,7 @@ class ShellEnvironmentTest(testtools.TestCase):
         if url:
             self.m.StubOutWithMock(request, 'urlopen')
             request.urlopen(url).AndReturn(six.BytesIO(content))
+            request.urlopen(url).AndReturn(six.BytesIO(content))
             self.m.ReplayAll()
 
         template_utils.resolve_environment_urls(
@@ -64,6 +65,8 @@ class ShellEnvironmentTest(testtools.TestCase):
             six.BytesIO(env))
         request.urlopen('file:///home/b/a.yaml').AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/a.yaml').AndReturn(
+            six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         files, env_dict = template_utils.process_environment_and_files(
@@ -89,6 +92,8 @@ class ShellEnvironmentTest(testtools.TestCase):
             six.BytesIO(env))
         request.urlopen('file:///home/my/dir/a.yaml').AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/my/dir/a.yaml').AndReturn(
+            six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         self.assertEqual(
@@ -122,6 +127,8 @@ class ShellEnvironmentTest(testtools.TestCase):
             six.BytesIO(env))
         request.urlopen('file:///home/my/bar/a.yaml').AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/my/bar/a.yaml').AndReturn(
+            six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         env_url = 'file://%s' % env_file
@@ -153,6 +160,7 @@ class ShellEnvironmentTest(testtools.TestCase):
         self.m.StubOutWithMock(request, 'urlopen')
         request.urlopen(url).AndReturn(six.BytesIO(env))
         request.urlopen(tmpl_url).AndReturn(six.BytesIO(self.template_a))
+        request.urlopen(tmpl_url).AndReturn(six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         files, env_dict = template_utils.process_environment_and_files(
@@ -205,10 +213,14 @@ class ShellEnvironmentTest(testtools.TestCase):
             six.BytesIO(env1))
         request.urlopen('file:///home/b/a.yaml').AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/a.yaml').AndReturn(
+            six.BytesIO(self.template_a))
         request.urlopen('file://%s' % env_file2).AndReturn(
             six.BytesIO(env2))
         request.urlopen('file:///home/b/b.yaml').AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/b.yaml').AndReturn(
+            six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         files, env = template_utils.process_multiple_environments_and_files(
@@ -251,18 +263,26 @@ class ShellEnvironmentTest(testtools.TestCase):
               "OS::Thingy4": "file:///home/b/b.yaml"
         '''
 
-        request.urlopen('file://%s' % env_file1).AndReturn(
+        request.urlopen('file://%s' % env_file1).InAnyOrder().AndReturn(
             six.BytesIO(env1))
         request.urlopen('file:///home/b/a.yaml').InAnyOrder().AndReturn(
             six.BytesIO(self.template_a))
         request.urlopen('file:///home/b/b.yaml').InAnyOrder().AndReturn(
             six.BytesIO(self.template_a))
-        request.urlopen('file://%s' % env_file2).AndReturn(
+        request.urlopen('file:///home/b/a.yaml').InAnyOrder().AndReturn(
+            six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/b.yaml').InAnyOrder().AndReturn(
+            six.BytesIO(self.template_a))
+        request.urlopen('file://%s' % env_file2).InAnyOrder().AndReturn(
             six.BytesIO(env2))
         request.urlopen('file:///home/b/a.yaml').InAnyOrder().AndReturn(
             six.BytesIO(self.template_a))
         request.urlopen('file:///home/b/b.yaml').InAnyOrder().AndReturn(
             six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/a.yaml').InAnyOrder().AndReturn(
+            six.BytesIO(self.template_a))
+        request.urlopen('file:///home/b/b.yaml').InAnyOrder().AndReturn(
+            six.BytesIO(self.template_a))
         self.m.ReplayAll()
 
         files, env = template_utils.process_multiple_environments_and_files(
@@ -718,11 +738,17 @@ parameters:
         request.urlopen(
             'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn(
                 six.BytesIO(self.foo_template))
+        request.urlopen(
+            'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn(
+                six.BytesIO(self.foo_template))
         request.urlopen(url).InAnyOrder().AndReturn(
             six.BytesIO(self.hot_template))
         request.urlopen(
             'file:///home/my/dir/spam/egg.yaml').InAnyOrder().AndReturn(
                 six.BytesIO(self.egg_template))
+        request.urlopen(
+            'file:///home/my/dir/spam/egg.yaml').InAnyOrder().AndReturn(
+                six.BytesIO(self.egg_template))
         self.m.ReplayAll()
 
         files, tmpl_parsed = template_utils.get_template_contents(
@@ -758,6 +784,90 @@ parameters:
         }, tmpl_parsed)
 
 
+class TestTemplateInFileFunctions(testtools.TestCase):
+
+    hot_template = b'''heat_template_version: 2013-05-23
+resources:
+  resource1:
+    type: OS::Heat::Stack
+    properties:
+      template: {get_file: foo.yaml}
+    '''
+
+    foo_template = b'''heat_template_version: "2013-05-23"
+resources:
+  foo:
+    type: OS::Type1
+    properties:
+      config: {get_file: bar.yaml}
+    '''
+
+    bar_template = b'''heat_template_version: "2013-05-23"
+parameters:
+  bar:
+    type: string
+    '''
+
+    def setUp(self):
+        super(TestTemplateInFileFunctions, self).setUp()
+        self.m = mox.Mox()
+
+        self.addCleanup(self.m.VerifyAll)
+        self.addCleanup(self.m.UnsetStubs)
+
+    def test_hot_template(self):
+        self.m.StubOutWithMock(request, 'urlopen')
+        tmpl_file = '/home/my/dir/template.yaml'
+        url = 'file:///home/my/dir/template.yaml'
+        foo_url = 'file:///home/my/dir/foo.yaml'
+        bar_url = 'file:///home/my/dir/bar.yaml'
+        request.urlopen(url).InAnyOrder().AndReturn(
+            six.BytesIO(self.hot_template))
+        request.urlopen(foo_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.foo_template))
+        request.urlopen(foo_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.foo_template))
+        request.urlopen(bar_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.bar_template))
+        request.urlopen(bar_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.bar_template))
+        self.m.ReplayAll()
+
+        files, tmpl_parsed = template_utils.get_template_contents(
+            template_file=tmpl_file)
+
+        self.assertEqual(yaml.load(self.bar_template.decode('utf-8')),
+                         json.loads(files.get('file:///home/my/dir/bar.yaml')))
+
+        self.assertEqual({
+            u'heat_template_version': u'2013-05-23',
+            u'resources': {
+                u'foo': {
+                    u'type': u'OS::Type1',
+                    u'properties': {
+                        u'config': {
+                            u'get_file': u'file:///home/my/dir/bar.yaml'
+                        }
+                    }
+                }
+            }
+        }, json.loads(files.get('file:///home/my/dir/foo.yaml')))
+
+        self.assertEqual({
+            u'heat_template_version': u'2013-05-23',
+            u'resources': {
+                u'resource1': {
+                    u'type': u'OS::Heat::Stack',
+                    u'properties': {
+                        u'template': {
+                            u'get_file': u'file:///home/my/dir/foo.yaml'
+                        }
+                    }
+                }
+            }
+        }, tmpl_parsed)
+
+
 class TestNestedIncludes(testtools.TestCase):
 
     hot_template = b'''heat_template_version: 2013-05-23
@@ -827,6 +937,8 @@ parameters:
             six.BytesIO(env))
         request.urlopen(template_url).AndReturn(
             six.BytesIO(self.hot_template))
+        request.urlopen(template_url).AndReturn(
+            six.BytesIO(self.hot_template))
 
         request.urlopen(foo_url).InAnyOrder().AndReturn(
             six.BytesIO(self.foo_template))
@@ -840,6 +952,15 @@ parameters:
             six.BytesIO(self.foo_template))
         request.urlopen(three_url).InAnyOrder().AndReturn(
             six.BytesIO(b'three contents'))
+        request.urlopen(foo_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.foo_template))
+        request.urlopen(egg_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.egg_template))
+        request.urlopen(one_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.foo_template))
+        request.urlopen(two_url).InAnyOrder().AndReturn(
+            six.BytesIO(self.foo_template))
+
         self.m.ReplayAll()
 
         files, env_dict = template_utils.process_environment_and_files(