Adding Job Templates support to CLI

Adding Job Templates commands to Sahara
OpenstackClient plugin:

$ dataprocessing job template create
$ dataprocessing job template list
$ dataprocessing job template show
$ dataprocessing job template update
$ dataprocessing job template delete

Partially implements: blueprint cli-as-openstackclient-plugin

Change-Id: I8c93b26c6eca37dbb144d7184c4d07dbe15bd9ab

2 files changed, 593 insertions, 0 deletions

diff --git a/saharaclient/osc/v1/job_templates.py b/saharaclient/osc/v1/job_templates.py
new file mode 100644
index 0000000..359bbb5
--- /dev/null
+++ b/saharaclient/osc/v1/job_templates.py
@@ -0,0 +1,323 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from cliff import command
+from cliff import lister
+from cliff import show
+from openstackclient.common import exceptions
+from openstackclient.common import utils as osc_utils
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+
+from saharaclient.osc.v1 import utils
+
+JOB_TEMPLATE_FIELDS = ['name', 'id', 'type', 'mains', 'libs', 'description',
+                       'is_public', 'is_protected']
+
+JOB_TYPES_CHOICES = ['Hive', 'Java', 'MapReduce', 'Storm', 'Pig', 'Shell',
+                     'MapReduce.Streaming', 'Spark']
+
+
+def _format_job_template_output(data):
+    data['mains'] = osc_utils.format_list(
+        ['%s:%s' % (m['name'], m['id']) for m in data['mains']])
+    data['libs'] = osc_utils.format_list(
+        ['%s:%s' % (l['name'], l['id']) for l in data['libs']])
+
+
+class CreateJobTemplate(show.ShowOne):
+    """Creates job template"""
+
+    log = logging.getLogger(__name__ + ".CreateJobTemplate")
+
+    def get_parser(self, prog_name):
+        parser = super(CreateJobTemplate, self).get_parser(prog_name)
+
+        parser.add_argument(
+            '--name',
+            metavar="<name>",
+            help="Name of the job template [REQUIRED if JSON is not provided]",
+        )
+        parser.add_argument(
+            '--type',
+            metavar="<type>",
+            choices=JOB_TYPES_CHOICES,
+            help="Type of the job (%s) "
+                 "[REQUIRED if JSON is not provided]" % ', '.join(
+                    JOB_TYPES_CHOICES)
+        )
+        parser.add_argument(
+            '--mains',
+            metavar="<main>",
+            nargs='+',
+            help="Name(s) or ID(s) for job's main job binary(s)",
+        )
+        parser.add_argument(
+            '--libs',
+            metavar="<lib>",
+            nargs='+',
+            help="Name(s) or ID(s) for job's lib job binary(s)",
+        )
+        parser.add_argument(
+            '--description',
+            metavar="<description>",
+            help="Description of the job template"
+        )
+        parser.add_argument(
+            '--public',
+            action='store_true',
+            default=False,
+            help='Make the job template public',
+        )
+        parser.add_argument(
+            '--protected',
+            action='store_true',
+            default=False,
+            help='Make the job template protected',
+        )
+        parser.add_argument(
+            '--interface',
+            metavar='<filename>',
+            help='JSON representation of the interface'
+        )
+        parser.add_argument(
+            '--json',
+            metavar='<filename>',
+            help='JSON representation of the job template'
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        client = self.app.client_manager.data_processing
+
+        if parsed_args.json:
+            blob = osc_utils.read_blob_file_contents(parsed_args.json)
+            try:
+                template = jsonutils.loads(blob)
+            except ValueError as e:
+                raise exceptions.CommandError(
+                    'An error occurred when reading '
+                    'template from file %s: %s' % (parsed_args.json, e))
+            data = client.jobs.create(**template).to_dict()
+        else:
+            if parsed_args.interface:
+                blob = osc_utils.read_blob_file_contents(parsed_args.json)
+                try:
+                    parsed_args.interface = jsonutils.loads(blob)
+                except ValueError as e:
+                    raise exceptions.CommandError(
+                        'An error occurred when reading '
+                        'interface from file %s: %s' % (parsed_args.json, e))
+
+            mains_ids = [utils.get_resource(client.job_binaries, m).id for m
+                         in parsed_args.mains] if parsed_args.mains else None
+            libs_ids = [utils.get_resource(client.job_binaries, m).id for m
+                        in parsed_args.libs] if parsed_args.libs else None
+
+            data = client.jobs.create(
+                name=parsed_args.name, type=parsed_args.type, mains=mains_ids,
+                libs=libs_ids, description=parsed_args.description,
+                interface=parsed_args.interface, is_public=parsed_args.public,
+                is_protected=parsed_args.protected).to_dict()
+
+        _format_job_template_output(data)
+        data = utils.prepare_data(data, JOB_TEMPLATE_FIELDS)
+
+        return self.dict2columns(data)
+
+
+class ListJobTemplates(lister.Lister):
+    """Lists job templates"""
+
+    log = logging.getLogger(__name__ + ".ListJobTemplates")
+
+    def get_parser(self, prog_name):
+        parser = super(ListJobTemplates, self).get_parser(prog_name)
+        parser.add_argument(
+            '--long',
+            action='store_true',
+            default=False,
+            help='List additional fields in output',
+        )
+        parser.add_argument(
+            '--type',
+            metavar="<type>",
+            choices=JOB_TYPES_CHOICES,
+            help="List job templates of specific type"
+        )
+        parser.add_argument(
+            '--name',
+            metavar="<name-substring>",
+            help="List job templates with specific substring in the "
+                 "name"
+        )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        client = self.app.client_manager.data_processing
+        search_opts = {'type': parsed_args.type} if parsed_args.type else {}
+
+        data = client.jobs.list(search_opts=search_opts)
+
+        if parsed_args.name:
+            data = utils.get_by_name_substring(data, parsed_args.name)
+
+        if parsed_args.long:
+            columns = ('name', 'id', 'type', 'description', 'is_public',
+                       'is_protected')
+            column_headers = utils.prepare_column_headers(columns)
+
+        else:
+            columns = ('name', 'id', 'type')
+            column_headers = utils.prepare_column_headers(columns)
+
+        return (
+            column_headers,
+            (osc_utils.get_item_properties(
+                s,
+                columns
+            ) for s in data)
+        )
+
+
+class ShowJobTemplate(show.ShowOne):
+    """Display job template details"""
+
+    log = logging.getLogger(__name__ + ".ShowJobTemplate")
+
+    def get_parser(self, prog_name):
+        parser = super(ShowJobTemplate, self).get_parser(prog_name)
+        parser.add_argument(
+            "job_template",
+            metavar="<job-template>",
+            help="Name or ID of the job template to display",
+        )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        client = self.app.client_manager.data_processing
+
+        data = utils.get_resource(
+            client.jobs, parsed_args.job_template).to_dict()
+
+        _format_job_template_output(data)
+        data = utils.prepare_data(data, JOB_TEMPLATE_FIELDS)
+
+        return self.dict2columns(data)
+
+
+class DeleteJobTemplate(command.Command):
+    """Deletes job template"""
+
+    log = logging.getLogger(__name__ + ".DeleteJobTemplate")
+
+    def get_parser(self, prog_name):
+        parser = super(DeleteJobTemplate, self).get_parser(prog_name)
+        parser.add_argument(
+            "job_template",
+            metavar="<job-template>",
+            nargs="+",
+            help="Name(s) or id(s) of the job template(s) to delete",
+        )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        client = self.app.client_manager.data_processing
+        for jt in parsed_args.job_template:
+            jt_id = utils.get_resource(
+                client.jobs, jt).id
+            client.jobs.delete(jt_id)
+
+
+class UpdateJobTemplate(show.ShowOne):
+    """Updates job template"""
+
+    log = logging.getLogger(__name__ + ".UpdateJobTemplate")
+
+    def get_parser(self, prog_name):
+        parser = super(UpdateJobTemplate, self).get_parser(prog_name)
+
+        parser.add_argument(
+            'job_template',
+            metavar="<job-template>",
+            help="Name or ID of the job template",
+        )
+        parser.add_argument(
+            '--name',
+            metavar="<name>",
+            help="New name of the job template",
+        )
+        parser.add_argument(
+            '--description',
+            metavar="<description>",
+            help='Description of the job template'
+        )
+        public = parser.add_mutually_exclusive_group()
+        public.add_argument(
+            '--public',
+            action='store_true',
+            help='Make the job template public '
+                 '(Visible from other tenants)',
+            dest='is_public'
+        )
+        public.add_argument(
+            '--private',
+            action='store_false',
+            help='Make the job_template private '
+                 '(Visible only from this tenant)',
+            dest='is_public'
+        )
+        protected = parser.add_mutually_exclusive_group()
+        protected.add_argument(
+            '--protected',
+            action='store_true',
+            help='Make the job template protected',
+            dest='is_protected'
+        )
+        protected.add_argument(
+            '--unprotected',
+            action='store_false',
+            help='Make the job template unprotected',
+            dest='is_protected'
+        )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        client = self.app.client_manager.data_processing
+
+        jt_id = utils.get_resource(
+            client.jobs, parsed_args.job_template).id
+
+        data = client.jobs.update(
+            jt_id,
+            name=parsed_args.name,
+            description=parsed_args.description,
+            is_public=parsed_args.is_public,
+            is_protected=parsed_args.is_protected
+        ).job
+
+        _format_job_template_output(data)
+        data = utils.prepare_data(data, JOB_TEMPLATE_FIELDS)
+
+        return self.dict2columns(data)
diff --git a/saharaclient/tests/unit/osc/v1/test_job_templates.py b/saharaclient/tests/unit/osc/v1/test_job_templates.py
new file mode 100644
index 0000000..5e716d0
--- /dev/null
+++ b/saharaclient/tests/unit/osc/v1/test_job_templates.py
@@ -0,0 +1,270 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from saharaclient.api import jobs as api_j
+from saharaclient.osc.v1 import job_templates as osc_j
+from saharaclient.tests.unit.osc.v1 import fakes
+
+JOB_INFO = {
+    "is_public": False,
+    "id": "job_id",
+    "name": "pig-job",
+    "description": "Job for test",
+    "interface": [],
+    "libs": [
+        {
+            "id": "lib_id",
+            "name": "lib"
+        }
+    ],
+    "type": "Pig",
+    "is_protected": False,
+    "mains": [
+        {
+            "id": "main_id",
+            "name": "main"
+        }
+    ]
+}
+
+
+class TestJobTemplates(fakes.TestDataProcessing):
+    def setUp(self):
+        super(TestJobTemplates, self).setUp()
+        self.job_mock = self.app.client_manager.data_processing.jobs
+        self.job_mock.reset_mock()
+
+
+class TestCreateJobTemplate(TestJobTemplates):
+    # TODO(apavlov): check for creation with --interface
+    def setUp(self):
+        super(TestCreateJobTemplate, self).setUp()
+        self.job_mock.create.return_value = api_j.Job(
+            None, JOB_INFO)
+        self.jb_mock = self.app.client_manager.data_processing.job_binaries
+        self.jb_mock.find_unique.return_value = mock.Mock(id='jb_id')
+        self.jb_mock.reset_mock()
+
+        # Command to test
+        self.cmd = osc_j.CreateJobTemplate(self.app, None)
+
+    def test_job_template_create_minimum_options(self):
+        arglist = ['--name', 'pig-job', '--type', 'Pig']
+        verifylist = [('name', 'pig-job'), ('type', 'Pig')]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.create.assert_called_once_with(
+            description=None, interface=None, is_protected=False,
+            is_public=False, libs=None, mains=None, name='pig-job', type='Pig')
+
+    def test_job_template_create_all_options(self):
+        arglist = ['--name', 'pig-job', '--type', 'Pig', '--mains', 'main',
+                   '--libs', 'lib', '--description', 'descr', '--public',
+                   '--protected']
+
+        verifylist = [('name', 'pig-job'), ('type', 'Pig'),
+                      ('mains', ['main']), ('libs', ['lib']),
+                      ('description', 'descr'), ('public', True),
+                      ('protected', True)]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.create.assert_called_once_with(
+            description='descr', interface=None, is_protected=True,
+            is_public=True, libs=['jb_id'], mains=['jb_id'], name='pig-job',
+            type='Pig')
+
+        # Check that columns are correct
+        expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
+                            'Libs', 'Mains', 'Name', 'Type')
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = ('Job for test', 'job_id', False, False, 'lib:lib_id',
+                         'main:main_id', 'pig-job', 'Pig')
+        self.assertEqual(expected_data, data)
+
+
+class TestListJobTemplates(TestJobTemplates):
+    def setUp(self):
+        super(TestListJobTemplates, self).setUp()
+        self.job_mock.list.return_value = [api_j.Job(
+            None, JOB_INFO)]
+
+        # Command to test
+        self.cmd = osc_j.ListJobTemplates(self.app, None)
+
+    def test_job_templates_list_no_options(self):
+        arglist = []
+        verifylist = []
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that columns are correct
+        expected_columns = ['Name', 'Id', 'Type']
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = [('pig-job', 'job_id', 'Pig')]
+        self.assertEqual(expected_data, list(data))
+
+    def test_job_template_list_long(self):
+        arglist = ['--long']
+        verifylist = [('long', True)]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that columns are correct
+        expected_columns = ['Name', 'Id', 'Type', 'Description', 'Is public',
+                            'Is protected']
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = [('pig-job', 'job_id', 'Pig', 'Job for test',
+                          False, False)]
+        self.assertEqual(expected_data, list(data))
+
+    def test_job_template_list_extra_search_opts(self):
+        arglist = ['--type', 'Pig', '--name', 'pig']
+        verifylist = [('type', 'Pig'), ('name', 'pig')]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that columns are correct
+        expected_columns = ['Name', 'Id', 'Type']
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = [('pig-job', 'job_id', 'Pig')]
+        self.assertEqual(expected_data, list(data))
+
+
+class TestShowJobTemplate(TestJobTemplates):
+    def setUp(self):
+        super(TestShowJobTemplate, self).setUp()
+        self.job_mock.find_unique.return_value = api_j.Job(
+            None, JOB_INFO)
+
+        # Command to test
+        self.cmd = osc_j.ShowJobTemplate(self.app, None)
+
+    def test_job_template_show(self):
+        arglist = ['pig-job']
+        verifylist = [('job_template', 'pig-job')]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.find_unique.assert_called_once_with(name='pig-job')
+
+        # Check that columns are correct
+        expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
+                            'Libs', 'Mains', 'Name', 'Type')
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = ('Job for test', 'job_id', False, False, 'lib:lib_id',
+                         'main:main_id', 'pig-job', 'Pig')
+        self.assertEqual(expected_data, data)
+
+
+class TestDeleteJobTemplate(TestJobTemplates):
+    def setUp(self):
+        super(TestDeleteJobTemplate, self).setUp()
+        self.job_mock.find_unique.return_value = api_j.Job(
+            None, JOB_INFO)
+
+        # Command to test
+        self.cmd = osc_j.DeleteJobTemplate(self.app, None)
+
+    def test_job_template_delete(self):
+        arglist = ['pig-job']
+        verifylist = [('job_template', ['pig-job'])]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.delete.assert_called_once_with('job_id')
+
+
+class TestUpdateJobTemplate(TestJobTemplates):
+    def setUp(self):
+        super(TestUpdateJobTemplate, self).setUp()
+        self.job_mock.find_unique.return_value = api_j.Job(None, JOB_INFO)
+        self.job_mock.update.return_value = mock.Mock(job=JOB_INFO.copy())
+
+        # Command to test
+        self.cmd = osc_j.UpdateJobTemplate(self.app, None)
+
+    def test_job_template_update_all_options(self):
+        arglist = ['pig-job', '--name', 'pig-job', '--description', 'descr',
+                   '--public', '--protected']
+
+        verifylist = [('job_template', 'pig-job'), ('name', 'pig-job'),
+                      ('description', 'descr'), ('is_public', True),
+                      ('is_protected', True)]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.update.assert_called_once_with(
+            'job_id', description='descr', is_protected=True, is_public=True,
+            name='pig-job')
+
+        # Check that columns are correct
+        expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
+                            'Libs', 'Mains', 'Name', 'Type')
+        self.assertEqual(expected_columns, columns)
+
+        # Check that data is correct
+        expected_data = ('Job for test', 'job_id', False, False, 'lib:lib_id',
+                         'main:main_id', 'pig-job', 'Pig')
+        self.assertEqual(expected_data, data)
+
+    def test_job_template_update_private_unprotected(self):
+        arglist = ['pig-job', '--private', '--unprotected']
+
+        verifylist = [('job_template', 'pig-job'), ('is_public', False),
+                      ('is_protected', False)]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        # Check that correct arguments were passed
+        self.job_mock.update.assert_called_once_with(
+            'job_id', description=None, is_protected=False, is_public=False,
+            name=None)