Add script for packaging a directory.

Mostly 7zips the directory, all files must already be in place.
On Linux the script also fixes the Qt related RPATHs.

Changes the RPATH fixing to only change the Qt RPATH,
instead of overwriting the complete RPATH to something custom.

Change-Id: I1e07b7c0fcb4292197a4e743782a1392f6558747
Reviewed-by: Eike Ziller <eike.ziller@theqtcompany.com>

3 files changed, 264 insertions, 68 deletions

diff --git a/scripts/common.py b/scripts/common.py
new file mode 100644
index 0000000000..6b96504acb
--- /dev/null
+++ b/scripts/common.py
@@ -0,0 +1,164 @@
+#############################################################################
+##
+## Copyright (C) 2015 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing
+##
+## This file is part of Qt Creator.
+##
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company.  For licensing terms and
+## conditions see http://www.qt.io/terms-conditions.  For further information
+## use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## In addition, as a special exception, The Qt Company gives you certain additional
+## rights.  These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+#############################################################################
+
+import os
+import shutil
+import subprocess
+import sys
+
+def is_windows_platform():
+    return sys.platform.startswith('win')
+
+def is_linux_platform():
+    return sys.platform.startswith('linux')
+
+def is_mac_platform():
+    return sys.platform.startswith('darwin')
+
+# copy of shutil.copytree that does not bail out if the target directory already exists
+# and that does not create empty directories
+def copytree(src, dst, symlinks=False, ignore=None):
+    def ensure_dir(destdir, ensure):
+        if ensure and not os.path.isdir(destdir):
+            os.makedirs(destdir)
+        return False
+
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    needs_ensure_dest_dir = True
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if symlinks and os.path.islink(srcname):
+                needs_ensure_dest_dir = ensure_dir(dst, needs_ensure_dest_dir)
+                linkto = os.readlink(srcname)
+                os.symlink(linkto, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore)
+            else:
+                needs_ensure_dest_dir = ensure_dir(dst, needs_ensure_dest_dir)
+                shutil.copy2(srcname, dstname)
+            # XXX What about devices, sockets etc.?
+        except (IOError, os.error) as why:
+            errors.append((srcname, dstname, str(why)))
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+    try:
+        if os.path.exists(dst):
+            shutil.copystat(src, dst)
+    except shutil.WindowsError:
+        # can't copy file access times on Windows
+        pass
+    except OSError as why:
+        errors.extend((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+
+def get_qt_install_info(qmake_bin):
+    output = subprocess.check_output([qmake_bin, '-query'])
+    lines = output.strip().split('\n')
+    info = {}
+    for line in lines:
+        (var, sep, value) = line.partition(':')
+        info[var.strip()] = value.strip()
+    return info
+
+def get_rpath(libfilepath, chrpath=None):
+    if chrpath is None:
+        chrpath = 'chrpath'
+    try:
+        output = subprocess.check_output([chrpath, '-l', libfilepath]).strip()
+    except subprocess.CalledProcessError: # no RPATH or RUNPATH
+        return []
+    marker = 'RPATH='
+    index = output.find(marker)
+    if index < 0:
+        marker = 'RUNPATH='
+        index = output.find(marker)
+    if index < 0:
+        return []
+    return output[index + len(marker):].split(':')
+
+def fix_rpaths(path, qt_deploy_path, qt_install_info, chrpath=None):
+    if chrpath is None:
+        chrpath = 'chrpath'
+    qt_install_prefix = qt_install_info['QT_INSTALL_PREFIX']
+    qt_install_libs = qt_install_info['QT_INSTALL_LIBS']
+
+    def fix_rpaths_helper(filepath):
+        rpath = get_rpath(filepath, chrpath)
+        if len(rpath) <= 0:
+            return
+        # remove previous Qt RPATH
+        new_rpath = filter(lambda path: not path.startswith(qt_install_prefix) and not path.startswith(qt_install_libs),
+                           rpath)
+
+        # add Qt RPATH if necessary
+        relative_path = os.path.relpath(qt_deploy_path, os.path.dirname(filepath))
+        if relative_path == '.':
+            relative_path = ''
+        else:
+            relative_path = '/' + relative_path
+        qt_rpath = '$ORIGIN' + relative_path
+        if not any((path == qt_rpath) for path in rpath):
+            new_rpath.append(qt_rpath)
+
+        # change RPATH
+        if len(new_rpath) > 0:
+            subprocess.check_call([chrpath, '-r', ':'.join(new_rpath), filepath])
+        else: # no RPATH / RUNPATH left. delete.
+            subprocess.check_call([chrpath, '-d', filepath])
+
+    def is_unix_executable(filepath):
+        # Whether a file is really a binary executable and not a script and not a symlink (unix only)
+        if os.path.exists(filepath) and os.access(filepath, os.X_OK) and not os.path.islink(filepath):
+            with open(filepath) as f:
+                return f.read(2) != "#!"
+
+    def is_unix_library(filepath):
+        # Whether a file is really a library and not a symlink (unix only)
+        return os.path.basename(filepath).find('.so') != -1 and not os.path.islink(filepath)
+
+    for dirpath, dirnames, filenames in os.walk(path):
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            if is_unix_executable(filepath) or is_unix_library(filepath):
+                fix_rpaths_helper(filepath)
+
diff --git a/scripts/deployqt.py b/scripts/deployqt.py
index 1ddbf61d36..6c89456620 100755
--- a/scripts/deployqt.py
+++ b/scripts/deployqt.py
@@ -36,6 +36,8 @@ import string
 import shutil
 from glob import glob
 
+import common
+
 ignoreErrors = False
 debug_build = False
 
@@ -50,7 +52,7 @@ def which(program):
     if fpath:
         if is_exe(program):
             return program
-        if sys.platform.startswith('win'):
+        if common.is_windows_platform():
             if is_exe(program + ".exe"):
                 return program  + ".exe"
     else:
@@ -58,7 +60,7 @@ def which(program):
             exe_file = os.path.join(path, program)
             if is_exe(exe_file):
                 return exe_file
-            if sys.platform.startswith('win'):
+            if common.is_windows_platform():
                 if is_exe(exe_file + ".exe"):
                     return exe_file  + ".exe"
 
@@ -86,44 +88,6 @@ def op_failed(details = None):
     else:
         print("Error: operation failed, but proceeding gracefully.")
 
-def fix_rpaths_helper(chrpath_bin, install_dir, dirpath, filenames):
-    # patch file
-    for filename in filenames:
-        fpath = os.path.join(dirpath, filename)
-        relpath = os.path.relpath(install_dir+'/lib/qtcreator', dirpath)
-        relpluginpath = os.path.relpath(install_dir+'/lib/qtcreator/plugins', dirpath)
-        command = [chrpath_bin, '-r', '$ORIGIN/'+relpath+':$ORIGIN/'+relpluginpath, fpath]
-        print fpath, ':', command
-        try:
-            subprocess.check_call(command)
-        except:
-            op_failed()
-
-def check_unix_binary_exec_helper(dirpath, filename):
-    """ Whether a file is really a binary executable and not a script (unix only)"""
-    fpath = os.path.join(dirpath, filename)
-    if os.path.exists(fpath) and os.access(fpath, os.X_OK):
-        with open(fpath) as f:
-            return f.read(2) != "#!"
-
-def check_unix_library_helper(dirpath, filename):
-    """ Whether a file is really a library and not a symlink (unix only)"""
-    fpath = os.path.join(dirpath, filename)
-    return filename.find('.so') != -1 and not os.path.islink(fpath)
-
-def fix_rpaths(chrpath_bin, install_dir):
-    print "fixing rpaths..."
-    for dirpath, dirnames, filenames in os.walk(os.path.join(install_dir, 'bin')):
-        #TODO remove library_helper once all libs moved out of bin/ on linux
-        filenames = [filename for filename in filenames if check_unix_binary_exec_helper(dirpath, filename) or check_unix_library_helper(dirpath, filename)]
-        fix_rpaths_helper(chrpath_bin, install_dir, dirpath, filenames)
-    for dirpath, dirnames, filenames in os.walk(os.path.join(install_dir, 'libexec', 'qtcreator')):
-        filenames = [filename for filename in filenames if check_unix_binary_exec_helper(dirpath, filename)]
-        fix_rpaths_helper(chrpath_bin, install_dir, dirpath, filenames)
-    for dirpath, dirnames, filenames in os.walk(os.path.join(install_dir, 'lib')):
-        filenames = [filename for filename in filenames if check_unix_library_helper(dirpath, filename)]
-        fix_rpaths_helper(chrpath_bin, install_dir, dirpath, filenames)
-
 def windows_debug_files_filter(filename):
     ignore_patterns = ['.lib', '.pdb', '.exp', '.ilk']
     for ip in ignore_patterns:
@@ -132,7 +96,7 @@ def windows_debug_files_filter(filename):
     return False
 
 def copy_ignore_patterns_helper(dir, filenames):
-    if not sys.platform.startswith('win'):
+    if not common.is_windows_platform:
         return filenames
 
     if debug_build:
@@ -146,17 +110,17 @@ def copy_ignore_patterns_helper(dir, filenames):
 def copy_qt_libs(install_dir, qt_libs_dir, qt_plugin_dir, qt_import_dir, qt_qml_dir, plugins, imports):
     print "copying Qt libraries..."
 
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
         libraries = glob(os.path.join(qt_libs_dir, '*.dll'))
     else:
         libraries = glob(os.path.join(qt_libs_dir, '*.so.*'))
 
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
         dest = os.path.join(install_dir, 'bin')
     else:
         dest = os.path.join(install_dir, 'lib', 'qtcreator')
 
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
         if debug_build:
             libraries = filter(lambda library: is_debug(library), libraries)
         else:
@@ -174,7 +138,7 @@ def copy_qt_libs(install_dir, qt_libs_dir, qt_plugin_dir, qt_import_dir, qt_qml_
             shutil.copy(library, dest)
 
     copy_ignore_func = None
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
         copy_ignore_func = copy_ignore_patterns_helper
 
     print "Copying plugins:", plugins
@@ -184,7 +148,7 @@ def copy_qt_libs(install_dir, qt_libs_dir, qt_plugin_dir, qt_import_dir, qt_qml_
             shutil.rmtree(target)
         pluginPath = os.path.join(qt_plugin_dir, plugin)
         if (os.path.exists(pluginPath)):
-            shutil.copytree(pluginPath, target, ignore=copy_ignore_func, symlinks=True)
+            common.copytree(pluginPath, target, ignore=copy_ignore_func, symlinks=True)
 
     print "Copying imports:", imports
     for qtimport in imports:
@@ -193,14 +157,14 @@ def copy_qt_libs(install_dir, qt_libs_dir, qt_plugin_dir, qt_import_dir, qt_qml_
             shutil.rmtree(target)
         import_path = os.path.join(qt_import_dir, qtimport)
         if os.path.exists(import_path):
-            shutil.copytree(import_path, target, ignore=copy_ignore_func, symlinks=True)
+            common.copytree(import_path, target, ignore=copy_ignore_func, symlinks=True)
 
     if (os.path.exists(qt_qml_dir)):
         print "Copying qt quick 2 imports"
         target = os.path.join(install_dir, 'bin', 'qml')
         if (os.path.exists(target)):
             shutil.rmtree(target)
-        shutil.copytree(qt_qml_dir, target, ignore=copy_ignore_func, symlinks=True)
+        common.copytree(qt_qml_dir, target, ignore=copy_ignore_func, symlinks=True)
 
 def add_qt_conf(target_dir, install_dir):
     qtconf_filepath = os.path.join(target_dir, 'qt.conf')
@@ -232,10 +196,10 @@ def copyPreservingLinks(source, destination):
     else:
         shutil.copy(source, destination)
 
-def copy_libclang(install_dir, llvm_install_dir):
+def deploy_libclang(install_dir, llvm_install_dir, chrpath_bin):
     # contains pairs of (source, target directory)
     deployinfo = []
-    if sys.platform.startswith("win"):
+    if common.is_windows_platform():
         deployinfo.append((os.path.join(llvm_install_dir, 'bin', 'libclang.dll'),
                            os.path.join(install_dir, 'bin')))
         deployinfo.append((os.path.join(llvm_install_dir, 'bin', 'clang.exe'),
@@ -262,14 +226,20 @@ def copy_libclang(install_dir, llvm_install_dir):
     for source, target in deployinfo:
         print source, '->', target
         copyPreservingLinks(source, target)
+
+    if common.is_linux_platform():
+        # libclang was statically compiled, so there is no need for the RPATHs
+        # and they are confusing when fixing RPATHs later in the process
+        print "removing libclang RPATHs..."
+        for source, target in deployinfo:
+            if not os.path.islink(target):
+                targetfilepath = target if not os.path.isdir(target) else os.path.join(target, os.path.basename(source))
+                subprocess.check_call([chrpath_bin, '-d', targetfilepath])
+
     print resourcesource, '->', resourcetarget
     if (os.path.exists(resourcetarget)):
         shutil.rmtree(resourcetarget)
-    shutil.copytree(resourcesource, resourcetarget, symlinks=True)
-
-def readQmakeVar(qmake_bin, var):
-    pipe = os.popen(' '.join([qmake_bin, '-query', var]))
-    return pipe.read().rstrip('\n')
+    common.copytree(resourcesource, resourcetarget, symlinks=True)
 
 def main():
     try:
@@ -301,41 +271,45 @@ def main():
         print "Cannot find required binary 'qmake'."
         sys.exit(2)
 
-    if not sys.platform.startswith('win'):
+    chrpath_bin = None
+    if common.is_linux_platform():
         chrpath_bin = which('chrpath')
         if chrpath_bin == None:
             print "Cannot find required binary 'chrpath'."
             sys.exit(2)
 
-    QT_INSTALL_LIBS = readQmakeVar(qmake_bin, 'QT_INSTALL_LIBS')
-    QT_INSTALL_BINS = readQmakeVar(qmake_bin, 'QT_INSTALL_BINS')
-    QT_INSTALL_PLUGINS = readQmakeVar(qmake_bin, 'QT_INSTALL_PLUGINS')
-    QT_INSTALL_IMPORTS = readQmakeVar(qmake_bin, 'QT_INSTALL_IMPORTS')
-    QT_INSTALL_QML = readQmakeVar(qmake_bin, 'QT_INSTALL_QML')
-    QT_INSTALL_TRANSLATIONS = readQmakeVar(qmake_bin, 'QT_INSTALL_TRANSLATIONS')
+    qt_install_info = common.get_qt_install_info(qmake_bin)
+    QT_INSTALL_LIBS = qt_install_info['QT_INSTALL_LIBS']
+    QT_INSTALL_BINS = qt_install_info['QT_INSTALL_BINS']
+    QT_INSTALL_PLUGINS = qt_install_info['QT_INSTALL_PLUGINS']
+    QT_INSTALL_IMPORTS = qt_install_info['QT_INSTALL_IMPORTS']
+    QT_INSTALL_QML = qt_install_info['QT_INSTALL_QML']
+    QT_INSTALL_TRANSLATIONS = qt_install_info['QT_INSTALL_TRANSLATIONS']
 
     plugins = ['accessible', 'codecs', 'designer', 'iconengines', 'imageformats', 'platformthemes', 'platforminputcontexts', 'platforms', 'printsupport', 'sqldrivers', 'xcbglintegrations']
     imports = ['Qt', 'QtWebKit']
 
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
         global debug_build
         debug_build = is_debug_build(install_dir)
 
-    if sys.platform.startswith('win'):
+    if common.is_windows_platform():
       copy_qt_libs(install_dir, QT_INSTALL_BINS, QT_INSTALL_PLUGINS, QT_INSTALL_IMPORTS, QT_INSTALL_QML, plugins, imports)
     else:
       copy_qt_libs(install_dir, QT_INSTALL_LIBS, QT_INSTALL_PLUGINS, QT_INSTALL_IMPORTS, QT_INSTALL_QML, plugins, imports)
     copy_translations(install_dir, QT_INSTALL_TRANSLATIONS)
     if "LLVM_INSTALL_DIR" in os.environ:
-      copy_libclang(install_dir, os.environ["LLVM_INSTALL_DIR"])
+      deploy_libclang(install_dir, os.environ["LLVM_INSTALL_DIR"], chrpath_bin)
 
-    if not sys.platform.startswith('win'):
-        fix_rpaths(chrpath_bin, install_dir)
+    if not common.is_windows_platform():
+        print "fixing rpaths..."
+        common.fix_rpaths(install_dir, os.path.join(install_dir, 'lib', 'qtcreator'),
+                          qt_install_info, chrpath_bin)
         add_qt_conf(os.path.join(install_dir, 'libexec', 'qtcreator'), install_dir) # e.g. for qml2puppet
     add_qt_conf(os.path.join(install_dir, 'bin'), install_dir)
 
 if __name__ == "__main__":
-    if sys.platform == 'darwin':
+    if common.is_mac_platform():
         print "Mac OS is not supported by this script, please use macqtdeploy!"
         sys.exit(2)
     else:
diff --git a/scripts/packagePlugins.py b/scripts/packagePlugins.py
new file mode 100755
index 0000000000..2f10cf931d
--- /dev/null
+++ b/scripts/packagePlugins.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+#############################################################################
+##
+## Copyright (C) 2015 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing
+##
+## This file is part of Qt Creator.
+##
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company.  For licensing terms and
+## conditions see http://www.qt.io/terms-conditions.  For further information
+## use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## In addition, as a special exception, The Qt Company gives you certain additional
+## rights.  These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+#############################################################################
+
+import argparse
+import os
+import subprocess
+import sys
+
+import common
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Deploy and 7z a directory of plugins.")
+    parser.add_argument('--7z', help='path to 7z binary',
+        default='7z.exe' if common.is_windows_platform() else '7z',
+        metavar='<7z_binary>', dest='sevenzip')
+    parser.add_argument('--qmake_binary', help='path to qmake binary which was used for compilation',
+        required=common.is_linux_platform(), metavar='<qmake_binary>')
+    parser.add_argument('source_directory', help='directory to deploy and 7z')
+    parser.add_argument('target_file', help='target file path of the resulting 7z')
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    arguments = parse_arguments()
+    if common.is_linux_platform():
+        qt_install_info = common.get_qt_install_info(arguments.qmake_binary)
+        common.fix_rpaths(arguments.source_directory,
+                          os.path.join(arguments.source_directory, 'lib', 'qtcreator'),
+                          qt_install_info)
+    subprocess.check_call([arguments.sevenzip, 'a', '-mx9', arguments.target_file,
+        os.path.join(arguments.source_directory, '*')])