utils: Extract 'url_directory_name' to a cython module

`url_directory_name` is heavily called from any downloadable source
plugin, and moving it to cython gives a more than 10x speedup

4 files changed, 48 insertions, 18 deletions

diff --git a/.pylintrc b/.pylintrc
index 63ff1b756..e07219a41 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,6 +7,7 @@ extension-pkg-whitelist=
     buildstream.node,
     buildstream._loader._loader,
     buildstream._loader.types,
+    buildstream._utils,
     buildstream._variables,
     buildstream._yaml
 
diff --git a/setup.py b/setup.py
index 1ea423e1a..284e74d7f 100755
--- a/setup.py
+++ b/setup.py
@@ -406,6 +406,7 @@ register_cython_module("buildstream.node")
 register_cython_module("buildstream._loader._loader")
 register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"])
 register_cython_module("buildstream._yaml", dependencies=["buildstream.node"])
+register_cython_module("buildstream._utils")
 register_cython_module("buildstream._variables", dependencies=["buildstream.node"])
 
 #####################################################
diff --git a/src/buildstream/_utils.pyx b/src/buildstream/_utils.pyx
new file mode 100644
index 000000000..797794eba
--- /dev/null
+++ b/src/buildstream/_utils.pyx
@@ -0,0 +1,42 @@
+#
+#  Copyright (C) 2019 Bloomberg L.P.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Benjamin Schubert <bschubert15@bloomberg.net>
+#
+
+"""
+This module contains utilities that have been optimized in Cython
+"""
+
+import string
+
+cdef str VALID_DIRECTORY_CHARS = string.digits + string.ascii_letters + "%_"
+
+
+def url_directory_name(str url):
+    """Normalizes a url into a directory name
+
+    Args:
+       url (str): A url string
+
+    Returns:
+       A string which can be used as a directory name
+    """
+    def transl(x):
+        return x if x in VALID_DIRECTORY_CHARS else '_'
+
+    return ''.join([transl(x) for x in url])
\ No newline at end of file
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index 82cd4134b..2c57925d4 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -30,7 +30,6 @@ import shutil
 import signal
 import stat
 from stat import S_ISDIR
-import string
 import subprocess
 import tempfile
 import itertools
@@ -42,6 +41,10 @@ from . import _signals
 from ._exceptions import BstError, ErrorDomain
 from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 
+# Contains utils that have been rewritten in Cython for speed benefits
+# This makes them available when importing from utils
+from ._utils import url_directory_name  # pylint: disable=unused-import
+
 # The magic number for timestamps: 2011-11-11 11:11:11
 BST_ARBITRARY_TIMESTAMP = calendar.timegm([2011, 11, 11, 11, 11, 11])
 
@@ -456,23 +459,6 @@ def get_host_tool(name):
     return program_path
 
 
-def url_directory_name(url):
-    """Normalizes a url into a directory name
-
-    Args:
-       url (str): A url string
-
-    Returns:
-       A string which can be used as a directory name
-    """
-    valid_chars = string.digits + string.ascii_letters + '%_'
-
-    def transl(x):
-        return x if x in valid_chars else '_'
-
-    return ''.join([transl(x) for x in url])
-
-
 def get_bst_version():
     """Gets the major, minor release portion of the
     BuildStream version.