osxbundle: fix bundle creation with python3

there were several problems that had to be fixed because of differences
between python2 to python3:
- subprocess.check_output returned an unicode instead of a string
- filter() returns an iterator instead of a list
- recursion limit was reached

first two were fixed by explicitly converting to the needed type or
using the proper function invocation. third was fixed by changing the
recursive process_libraries function to an iterative one.

Fixes #5600, #3316

TOOLS/dylib-unhell.py

@@ -24,8 +24,8 @@ def is_user_lib(objfile, libname):
 
 def otool(objfile):
     command = "otool -L %s | grep -e '\t' | awk '{ print $1 }'" % objfile
-    output  = subprocess.check_output(command, shell = True)
-    return filter(partial(is_user_lib, objfile), output.split())
+    output  = subprocess.check_output(command, shell = True, universal_newlines=True)
+    return set(filter(partial(is_user_lib, objfile), output.split()))
 
 def install_name_tool_change(old, new, objfile):
     subprocess.call(["install_name_tool", "-change", old, new, objfile])
@@ -35,7 +35,7 @@ def install_name_tool_id(name, objfile):
 
 def libraries(objfile, result = dict()):
     libs_list       = otool(objfile)
-    result[objfile] = set(libs_list)
+    result[objfile] = libs_list
 
     for lib in libs_list:
         if lib not in result:
@@ -80,15 +80,17 @@ def process_libraries(libs_dict, binary, processed = []):
             if p in libs_dict[src]:
                 install_name_tool_change(p, lib_name(p), dst)
 
-    process_libraries(libs_dict, binary, ls)
+    return ls
 
 def main():
     binary = os.path.abspath(sys.argv[1])
     if not os.path.exists(lib_path(binary)):
         os.makedirs(lib_path(binary))
     libs = libraries(binary)
-    print(libs)
-    process_libraries(libs, binary)
+
+    libs_processed = process_libraries(libs, binary)
+    while libs_processed is not None:
+        libs_processed = process_libraries(libs, binary, libs_processed)
 
 if __name__ == "__main__":
     main()