osxbundle: split and optimize bundling script

Move the code that copies the dylib's to the bundle to a new script
(dylib-unhell.py) which is called by osxbundle.py.

dylib-unhell is about 20x faster than the previous implementation. This is
accomplished by removing superflous shell-out operations which are kept track
of using an in memory tree of all the needed dependencies. Moreover the
shell-outs have been further optimized by not requiring a complete shell for
every operation and just using subprocess.call (which is equivalent to Popen).

TOOLS/dylib-unhell.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import shutil
+import subprocess
+from functools import partial
+
+sys_re = re.compile("^/System")
+usr_re = re.compile("^/usr/lib/")
+exe_re = re.compile("@executable_path")
+
+def is_user_lib(objfile, libname):
+    return not sys_re.match(libname) and \
+           not usr_re.match(libname) and \
+           not exe_re.match(libname) and \
+           not "libobjc" in libname and \
+           not "libSystem" in libname and \
+           not "libc" in libname and \
+           not "libgcc" in libname and \
+           not os.path.basename(objfile) in libname
+
+def otool(objfile):
+    command = "otool -L %s | grep -e '\t' | awk '{ print $1 }'" % objfile
+    output  = subprocess.check_output(command, shell = True)
+    return filter(partial(is_user_lib, objfile), output.split())
+
+def install_name_tool_change(old, new, objfile):
+    subprocess.call(["install_name_tool", "-change", old, new, objfile])
+
+def install_name_tool_id(name, objfile):
+    subprocess.call(["install_name_tool", "-id", name, objfile])
+
+def libraries(objfile, result = dict()):
+    libs_list       = otool(objfile)
+    result[objfile] = set(libs_list)
+
+    for lib in libs_list:
+        if lib not in result:
+            libraries(lib, result)
+
+    return result
+
+def leafs(libs_dict, processed = []):
+    result    = []
+    processed = set(processed)
+
+    for objfile, libs in libs_dict.items():
+        if libs <= processed:
+            result.append(objfile)
+
+    return result
+
+def lib_path(binary):
+    return os.path.join(os.path.dirname(binary), 'lib')
+
+def lib_name(lib):
+    return os.path.join("@executable_path", "lib", os.path.basename(lib))
+
+def process_libraries(libs_dict, binary, processed = []):
+    ls   = leafs(libs_dict, processed)
+    diff = set(ls) - set(processed)
+    if diff == set():
+        return
+
+    for src in diff:
+        name = lib_name(src)
+        dst  = os.path.join(lib_path(binary), os.path.basename(src))
+
+        shutil.copy(src, dst)
+        os.chmod(dst, 0o755)
+        install_name_tool_id(name, dst)
+
+        if src in libs_dict[binary]:
+            install_name_tool_change(src, name, binary)
+
+        for p in processed:
+            if p in libs_dict[src]:
+                install_name_tool_change(p, lib_name(p), dst)
+
+    process_libraries(libs_dict, binary, ls)
+
+def main():
+    binary = os.path.abspath(sys.argv[1])
+    if not os.path.exists(lib_path(binary)):
+        os.makedirs(lib_path(binary))
+    libs = libraries(binary)
+    print(libs)
+    process_libraries(libs, binary)
+
+if __name__ == "__main__":
+    main()

TOOLS/osxbundle.py

@@ -1,35 +1,13 @@
 #!/usr/bin/env python
 
 import os
-import re
 import shutil
 import sys
 from optparse import OptionParser
-from textwrap import dedent
 
 def sh(command):
     return os.popen(command).read()
 
-def dylib_lst(input_file):
-    return sh("otool -L %s | grep -e '\t' | awk '{ print $1 }'" % input_file)
-
-sys_re = re.compile("/System")
-exe_re = re.compile("@executable_path")
-binary_name = sys.argv[1]
-
-def is_user_lib(libname, input_file):
-    return not sys_re.match(libname) and \
-           not exe_re.match(libname) and \
-           not "libobjc" in libname and \
-           not "libSystem" in libname and \
-           not "libgcc" in libname and \
-           not os.path.basename(input_file) in libname and \
-           not libname == ''
-
-def user_dylib_lst(input_file):
-    return [lib for lib in dylib_lst(input_file).split("\n") if
-            is_user_lib(lib, input_file)]
-
 def bundle_path(binary_name):
     return "%s.app" % binary_name
 
@@ -56,52 +34,9 @@ def copy_bundle(binary_name):
 def copy_binary(binary_name):
     shutil.copy(binary_name, target_binary(binary_name))
 
-def run_install_name_tool(target_file, dylib_path, dest_dir, root=True):
-    new_dylib_path = os.path.join("@executable_path", "lib",
-        os.path.basename(dylib_path))
-
-    sh("install_name_tool -change %s %s %s" % \
-        (dylib_path, new_dylib_path, target_file))
-    if root:
-        sh("install_name_tool -id %s %s" % \
-            (new_dylib_path, os.path.join(dest_dir,
-                os.path.basename(dylib_path))))
-
-def cp_dylibs(target_file, dest_dir):
-    for dylib_path in user_dylib_lst(target_file):
-        dylib_dest_path = os.path.join(dest_dir, os.path.basename(dylib_path))
-
-        try:
-            shutil.copy(dylib_path, dylib_dest_path)
-        except IOError:
-            if re.match("dylib$", target_file):
-                reinstall_what = target_file
-            else:
-                reinstall_what = dylib_path
-
-            sys.exit(dedent("""\
-                %s uses library %s which is not available anymore.
-                This is most likely because you uninstalled %s.
-                Please reinstall %s to fix it's dependencies.""" % \
-                   (target_file, dylib_path, dylib_path, reinstall_what) ))
-
-        os.chmod(dylib_dest_path, 0o755)
-        cp_dylibs(dylib_dest_path, dest_dir)
-
-def fix_dylibs_paths(target_file, dest_dir, root=True):
-    for dylib_path in user_dylib_lst(target_file):
-        dylib_dest_path = os.path.join(dest_dir, os.path.basename(dylib_path))
-        run_install_name_tool(target_file, dylib_path, dest_dir, root)
-        fix_dylibs_paths(dylib_dest_path, dest_dir, False)
-
 def apply_plist_template(plist_file, version):
     sh("sed -i -e 's/${VERSION}/%s/g' %s" % (version, plist_file))
 
-def bundle_dependencies(binary_name):
-    lib_bundle_directory = os.path.join(target_directory(binary_name), "lib")
-    cp_dylibs(binary_name, lib_bundle_directory)
-    fix_dylibs_paths(target_binary(binary_name), lib_bundle_directory)
-
 def main():
     version = sh("./version.sh --print").strip()
 
@@ -128,7 +63,7 @@ def main():
 
     if options.deps:
         print("> bundling dependencies")
-        bundle_dependencies(binary_name)
+        sh(" ".join(["TOOLS/dylib-unhell.py", target_binary(binary_name)]))
 
     print("done.")