Implements blueprint vnc-console-cleanup

 * Creates a unified way to access vnc consoles for xenserver and libvirt
 * Now supports both java and websocket clients
 * Removes nova-vncproxy - a replacement version of this (nova-novncproxy) can be found as described in vncconsole.rst
 * Adds nova-xvpvncproxy, which supports a java vnc client
 * Adds api extension to access java and novnc access_urls
 * Fixes proxy server to close/shutdown sockets more cleanly
 * Address style feedback
 * Use new-style extension format
 * Fix setup.py
 * utils.gen_uuid must be wrapped like str(utils.gen_uuid()) or it can't be serialized

Change-Id: I5e42e2f160e8e3476269bd64b0e8aa77e66c918c

5 files changed, 189 insertions, 371 deletions

diff --git a/nova/vnc/__init__.py b/nova/vnc/__init__.py
index 859bfd65f1..bfaf0b391b 100644
--- a/nova/vnc/__init__.py
+++ b/nova/vnc/__init__.py
@@ -22,13 +22,15 @@ from nova import flags
 
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('vncproxy_topic', 'vncproxy',
-                   'the topic vnc proxy nodes listen on')
-flags.DEFINE_string('vncproxy_url',
-                   'http://127.0.0.1:6080',
+flags.DEFINE_string('novncproxy_base_url',
+                   'http://127.0.0.1:6080/vnc_auto.html',
                    'location of vnc console proxy, \
-                    in the form "http://127.0.0.1:6080"')
-flags.DEFINE_string('vncserver_host', '0.0.0.0',
+                    in the form "http://127.0.0.1:6080/vnc_auto.html"')
+flags.DEFINE_string('xvpvncproxy_base_url',
+                   'http://127.0.0.1:6081/console',
+                   'location of nova xvp vnc console proxy, \
+                    in the form "http://127.0.0.1:6081/console"')
+flags.DEFINE_string('vncserver_host', '127.0.0.1',
                     'the host interface on which vnc server should listen')
 flags.DEFINE_bool('vnc_enabled', True,
                   'enable vnc related features')
diff --git a/nova/vnc/auth.py b/nova/vnc/auth.py
deleted file mode 100644
index b96dc595ee..0000000000
--- a/nova/vnc/auth.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright (c) 2010 Openstack, LLC.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-"""Auth Components for VNC Console."""
-
-import time
-import urlparse
-import webob
-
-from nova import context
-from nova import flags
-from nova import log as logging
-from nova import manager
-from nova import rpc
-from nova import utils
-from nova import vnc
-
-
-LOG = logging.getLogger('nova.vncproxy')
-FLAGS = flags.FLAGS
-
-
-class VNCNovaAuthMiddleware(object):
-    """Implementation of Middleware to Handle Nova Auth."""
-
-    def __init__(self, app):
-        self.app = app
-        self.token_cache = {}
-        utils.LoopingCall(self.delete_expired_cache_items).start(1)
-
-    @webob.dec.wsgify
-    def __call__(self, req):
-        token = req.params.get('token')
-
-        if not token:
-            referrer = req.environ.get('HTTP_REFERER')
-            auth_params = urlparse.parse_qs(urlparse.urlparse(referrer).query)
-            if 'token' in auth_params:
-                token = auth_params['token'][0]
-
-        connection_info = self.get_token_info(token)
-        if not connection_info:
-            LOG.audit(_("Unauthorized Access: (%s)"), req.environ)
-            return webob.exc.HTTPForbidden(detail='Unauthorized')
-
-        if req.path == vnc.proxy.WS_ENDPOINT:
-            req.environ['vnc_host'] = connection_info['host']
-            req.environ['vnc_port'] = int(connection_info['port'])
-
-        return req.get_response(self.app)
-
-    def get_token_info(self, token):
-        if token in self.token_cache:
-            return self.token_cache[token]
-
-        rval = rpc.call(context.get_admin_context(),
-                        FLAGS.vncproxy_topic,
-                        {"method": "check_token", "args": {'token': token}})
-        if rval:
-            self.token_cache[token] = rval
-        return rval
-
-    def delete_expired_cache_items(self):
-        now = time.time()
-        to_delete = []
-        for k, v in self.token_cache.items():
-            if now - v['last_activity_at'] > FLAGS.vnc_token_ttl:
-                to_delete.append(k)
-
-        for k in to_delete:
-            del self.token_cache[k]
-
-
-class LoggingMiddleware(object):
-    """Middleware for basic vnc-specific request logging."""
-
-    def __init__(self, app):
-        self.app = app
-
-    @webob.dec.wsgify
-    def __call__(self, req):
-        if req.path == vnc.proxy.WS_ENDPOINT:
-            LOG.info(_("Received Websocket Request: %s"), req.url)
-        else:
-            LOG.info(_("Received Request: %s"), req.url)
-
-        return req.get_response(self.app)
-
-
-class VNCProxyAuthManager(manager.Manager):
-    """Manages token based authentication."""
-
-    def __init__(self, scheduler_driver=None, *args, **kwargs):
-        super(VNCProxyAuthManager, self).__init__(*args, **kwargs)
-        self.tokens = {}
-        utils.LoopingCall(self._delete_expired_tokens).start(1)
-
-    def authorize_vnc_console(self, context, token, host, port):
-        self.tokens[token] = {'host': host,
-                              'port': port,
-                              'last_activity_at': time.time()}
-        token_dict = self.tokens[token]
-        LOG.audit(_("Received Token: %(token)s, %(token_dict)s)"), locals())
-
-    def check_token(self, context, token):
-        token_valid = token in self.tokens
-        LOG.audit(_("Checking Token: %(token)s, %(token_valid)s)"), locals())
-        if token_valid:
-            return self.tokens[token]
-
-    def _delete_expired_tokens(self):
-        now = time.time()
-        to_delete = []
-        for k, v in self.tokens.items():
-            if now - v['last_activity_at'] > FLAGS.vnc_token_ttl:
-                to_delete.append(k)
-
-        for k in to_delete:
-            LOG.audit(_("Deleting Expired Token: %s)"), k)
-            del self.tokens[k]
diff --git a/nova/vnc/proxy.py b/nova/vnc/proxy.py
deleted file mode 100644
index 376db40c15..0000000000
--- a/nova/vnc/proxy.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright (c) 2010 Openstack, LLC.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-"""Eventlet WSGI Services to proxy VNC.  No nova deps."""
-
-import base64
-import os
-
-import eventlet
-from eventlet import websocket
-
-import webob
-
-
-WS_ENDPOINT = '/data'
-
-
-class WebsocketVNCProxy(object):
-    """Class to proxy from websocket to vnc server."""
-
-    def __init__(self, wwwroot):
-        self.wwwroot = wwwroot
-        self.whitelist = {}
-        for root, dirs, files in os.walk(wwwroot):
-            hidden_dirs = []
-            for d in dirs:
-                if d.startswith('.'):
-                    hidden_dirs.append(d)
-            for d in hidden_dirs:
-                dirs.remove(d)
-            for name in files:
-                if not str(name).startswith('.'):
-                    filename = os.path.join(root, name)
-                    self.whitelist[filename] = True
-
-    def get_whitelist(self):
-        return self.whitelist.keys()
-
-    def sock2ws(self, source, dest):
-        try:
-            while True:
-                d = source.recv(32384)
-                if d == '':
-                    break
-                d = base64.b64encode(d)
-                dest.send(d)
-        except Exception:
-            source.close()
-            dest.close()
-
-    def ws2sock(self, source, dest):
-        try:
-            while True:
-                d = source.wait()
-                if d is None:
-                    break
-                d = base64.b64decode(d)
-                dest.sendall(d)
-        except Exception:
-            source.close()
-            dest.close()
-
-    def proxy_connection(self, environ, start_response):
-        @websocket.WebSocketWSGI
-        def _handle(client):
-            server = eventlet.connect((client.environ['vnc_host'],
-                                       client.environ['vnc_port']))
-            t1 = eventlet.spawn(self.ws2sock, client, server)
-            t2 = eventlet.spawn(self.sock2ws, server, client)
-            t1.wait()
-            t2.wait()
-        _handle(environ, start_response)
-
-    def __call__(self, environ, start_response):
-        req = webob.Request(environ)
-        if req.path == WS_ENDPOINT:
-            return self.proxy_connection(environ, start_response)
-        else:
-            if req.path == '/':
-                fname = '/vnc_auto.html'
-            else:
-                fname = req.path
-
-            fname = (self.wwwroot + fname).replace('//', '/')
-            if not fname in self.whitelist:
-                start_response('404 Not Found',
-                               [('content-type', 'text/html')])
-                return "Not Found"
-
-            base, ext = os.path.splitext(fname)
-            if ext == '.js':
-                mimetype = 'application/javascript'
-            elif ext == '.css':
-                mimetype = 'text/css'
-            elif ext in ['.svg', '.jpg', '.png', '.gif']:
-                mimetype = 'image'
-            else:
-                mimetype = 'text/html'
-
-            start_response('200 OK', [('content-type', mimetype)])
-            return open(os.path.join(fname)).read()
-
-
-class DebugMiddleware(object):
-    """Debug middleware.  Skip auth, get vnc connect info from query string."""
-
-    def __init__(self, app):
-        self.app = app
-
-    @webob.dec.wsgify
-    def __call__(self, req):
-        if req.path == WS_ENDPOINT:
-            req.environ['vnc_host'] = req.params.get('host')
-            req.environ['vnc_port'] = int(req.params.get('port'))
-        return req.get_response(self.app)
diff --git a/nova/vnc/server.py b/nova/vnc/server.py
deleted file mode 100644
index c6eb7020fc..0000000000
--- a/nova/vnc/server.py
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright (c) 2010 Openstack, LLC.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-"""Auth Components for VNC Console."""
-
-import os
-import sys
-
-from nova import flags
-from nova import log as logging
-from nova import version
-from nova import wsgi
-from nova.vnc import auth
-from nova.vnc import proxy
-
-
-LOG = logging.getLogger('nova.vncproxy')
-FLAGS = flags.FLAGS
-flags.DEFINE_string('vncproxy_wwwroot', '/var/lib/nova/noVNC/',
-                     'Full path to noVNC directory')
-flags.DEFINE_boolean('vnc_debug', False,
-                     'Enable debugging features, like token bypassing')
-flags.DEFINE_integer('vncproxy_port', 6080,
-                     'Port that the VNC proxy should bind to')
-flags.DEFINE_string('vncproxy_host', '0.0.0.0',
-                     'Address that the VNC proxy should bind to')
-flags.DEFINE_integer('vncproxy_flash_socket_policy_port', 843,
-                     'Port that the socket policy listener should bind to')
-flags.DEFINE_string('vncproxy_flash_socket_policy_host', '0.0.0.0',
-                     'Address that the socket policy listener should bind to')
-flags.DEFINE_integer('vnc_token_ttl', 300,
-                     'How many seconds before deleting tokens')
-flags.DEFINE_string('vncproxy_manager', 'nova.vnc.auth.VNCProxyAuthManager',
-                    'Manager for vncproxy auth')
-
-
-def get_wsgi_server():
-    LOG.audit(_("Starting nova-vncproxy node (version %s)"),
-              version.version_string_with_vcs())
-
-    if not (os.path.exists(FLAGS.vncproxy_wwwroot) and
-            os.path.exists(FLAGS.vncproxy_wwwroot + '/vnc_auto.html')):
-        LOG.info(_("Missing vncproxy_wwwroot (version %s)"),
-                    FLAGS.vncproxy_wwwroot)
-        LOG.info(_("You need a slightly modified version of noVNC "
-                   "to work with the nova-vnc-proxy"))
-        LOG.info(_("Check out the most recent nova noVNC code: %s"),
-                   "git://github.com/sleepsonthefloor/noVNC.git")
-        LOG.info(_("And drop it in %s"), FLAGS.vncproxy_wwwroot)
-        sys.exit(1)
-
-    app = proxy.WebsocketVNCProxy(FLAGS.vncproxy_wwwroot)
-
-    LOG.audit(_("Allowing access to the following files: %s"),
-              app.get_whitelist())
-
-    with_logging = auth.LoggingMiddleware(app)
-
-    if FLAGS.vnc_debug:
-        with_auth = proxy.DebugMiddleware(with_logging)
-    else:
-        with_auth = auth.VNCNovaAuthMiddleware(with_logging)
-
-    wsgi_server = wsgi.Server("VNC Proxy",
-                              with_auth,
-                              host=FLAGS.vncproxy_host,
-                              port=FLAGS.vncproxy_port)
-    wsgi_server.start_tcp(handle_flash_socket_policy,
-                          host=FLAGS.vncproxy_flash_socket_policy_host,
-                          port=FLAGS.vncproxy_flash_socket_policy_port)
-    return wsgi_server
-
-
-def handle_flash_socket_policy(socket):
-    LOG.info(_("Received connection on flash socket policy port"))
-
-    fd = socket.makefile('rw')
-    expected_command = "<policy-file-request/>"
-    if expected_command in fd.read(len(expected_command) + 1):
-        LOG.info(_("Received valid flash socket policy request"))
-        fd.write('<?xml version="1.0"?><cross-domain-policy><allow-'
-                 'access-from domain="*" to-ports="%d" /></cross-'
-                 'domain-policy>' % (FLAGS.vncproxy_port))
-        fd.flush()
-    socket.close()
diff --git a/nova/vnc/xvp_proxy.py b/nova/vnc/xvp_proxy.py
new file mode 100644
index 0000000000..fa1845726a
--- /dev/null
+++ b/nova/vnc/xvp_proxy.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2012 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""Eventlet WSGI Services to proxy VNC for XCP protocol."""
+
+import base64
+import os
+import socket
+import webob
+
+import eventlet
+import eventlet.green
+import eventlet.greenio
+import eventlet.wsgi
+
+from nova import context
+from nova import flags
+from nova import log as logging
+from nova import rpc
+from nova import version
+from nova import wsgi
+
+
+LOG = logging.getLogger('nova.xvpvncproxy')
+FLAGS = flags.FLAGS
+
+flags.DECLARE('consoleauth_topic', 'nova.consoleauth')
+flags.DEFINE_integer('xvpvncproxy_port', 6081,
+                     'Port that the XCP VNC proxy should bind to')
+flags.DEFINE_string('xvpvncproxy_host', '0.0.0.0',
+                     'Address that the XCP VNC proxy should bind to')
+
+
+class XCPVNCProxy(object):
+    """Class to use the xvp auth protocol to proxy instance vnc consoles."""
+
+    def one_way_proxy(self, source, dest):
+        """Proxy tcp connection from source to dest."""
+        while True:
+            try:
+                d = source.recv(32384)
+            except Exception as e:
+                d = None
+
+            # If recv fails, send a write shutdown the other direction
+            if d is None or len(d) == 0:
+                dest.shutdown(socket.SHUT_WR)
+                break
+            # If send fails, terminate proxy in both directions
+            try:
+                # sendall raises an exception on write error, unlike send
+                dest.sendall(d)
+            except Exception as e:
+                source.close()
+                dest.close()
+                break
+
+    def handshake(self, req, connect_info, sockets):
+        """Execute hypervisor-specific vnc auth handshaking (if needed)."""
+        host = connect_info['host']
+        port = int(connect_info['port'])
+
+        server = eventlet.connect((host, port))
+
+        # Handshake as necessary
+        if connect_info.get('internal_access_path'):
+            server.sendall("CONNECT %s HTTP/1.1\r\n\r\n" %
+                        connect_info['internal_access_path'])
+
+            data = ""
+            while True:
+                b = server.recv(1)
+                if b:
+                    data += b
+                    if data.find("\r\n\r\n") != -1:
+                        if not data.split("\r\n")[0].find("200"):
+                            LOG.audit(_("Error in handshake: %s"), data)
+                            return
+                        break
+
+                if not b or len(data) > 4096:
+                    LOG.audit(_("Error in handshake: %s"), data)
+                    return
+
+        client = req.environ['eventlet.input'].get_socket()
+        client.sendall("HTTP/1.1 200 OK\r\n\r\n")
+        socketsserver = None
+        sockets['client'] = client
+        sockets['server'] = server
+
+    def proxy_connection(self, req, connect_info):
+        """Spawn bi-directional vnc proxy."""
+        sockets = {}
+        t0 = eventlet.spawn(self.handshake, req, connect_info, sockets)
+        t0.wait()
+
+        if not sockets.get('client') or not sockets.get('server'):
+            LOG.audit(_("Invalid request: %s"), req)
+            start_response('400 Invalid Request',
+                           [('content-type', 'text/html')])
+            return "Invalid Request"
+
+        client = sockets['client']
+        server = sockets['server']
+
+        t1 = eventlet.spawn(self.one_way_proxy, client, server)
+        t2 = eventlet.spawn(self.one_way_proxy, server, client)
+        t1.wait()
+        t2.wait()
+
+        # Make sure our sockets are closed
+        server.close()
+        client.close()
+
+    def __call__(self, environ, start_response):
+        try:
+            req = webob.Request(environ)
+            LOG.audit(_("Request: %s"), req)
+            token = req.params.get('token')
+            if not token:
+                LOG.audit(_("Request made with missing token: %s"), req)
+                start_response('400 Invalid Request',
+                               [('content-type', 'text/html')])
+                return "Invalid Request"
+
+            ctxt = context.get_admin_context()
+            connect_info = rpc.call(ctxt, FLAGS.consoleauth_topic,
+                                    {'method': 'check_token',
+                                     'args': {'token': token}})
+
+            if not connect_info:
+                LOG.audit(_("Request made with invalid token: %s"), req)
+                start_response('401 Not Authorized',
+                               [('content-type', 'text/html')])
+                return "Not Authorized"
+
+            self.proxy_connection(req, connect_info)
+        except Exception as e:
+            LOG.audit(_("Unexpected error: %s"), e)
+
+
+class SafeHttpProtocol(eventlet.wsgi.HttpProtocol):
+    """HttpProtocol wrapper to suppress IOErrors.
+
+       The proxy code above always shuts down client connections, so we catch
+       the IOError that raises when the SocketServer tries to flush the
+       connection.
+    """
+    def finish(self):
+        try:
+            eventlet.green.BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
+        except IOError:
+            pass
+        eventlet.greenio.shutdown_safe(self.connection)
+        self.connection.close()
+
+
+def get_wsgi_server():
+    LOG.audit(_("Starting nova-xvpvncproxy node (version %s)"),
+              version.version_string_with_vcs())
+
+    return  wsgi.Server("XCP VNC Proxy",
+                         XCPVNCProxy(),
+                         protocol=SafeHttpProtocol,
+                         host=FLAGS.xvpvncproxy_host,
+                         port=FLAGS.xvpvncproxy_port)