#!/usr/bin/env python3 """ # OpenWrt download directory cleanup utility. # Delete all but the very last version of the program tarballs. # # Copyright (C) 2010-2015 Michael Buesch # Copyright (C) 2013-2015 OpenWrt.org """ from __future__ import print_function import sys import os import re import getopt import shutil # Commandline options opt_dryrun = False def parseVer_1234(match, filepath): progname = match.group(1) progversion = ( (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | (int(match.group(4)) << 32) | (int(match.group(5)) << 16) ) return (progname, progversion) def parseVer_123(match, filepath): progname = match.group(1) try: patchlevel = match.group(5) except IndexError as e: patchlevel = None if patchlevel: patchlevel = ord(patchlevel[0]) else: patchlevel = 0 progversion = ( (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | (int(match.group(4)) << 32) | patchlevel ) return (progname, progversion) def parseVer_12(match, filepath): progname = match.group(1) try: patchlevel = match.group(4) except IndexError as e: patchlevel = None if patchlevel: patchlevel = ord(patchlevel[0]) else: patchlevel = 0 progversion = (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | patchlevel return (progname, progversion) def parseVer_r(match, filepath): progname = match.group(1) progversion = int(match.group(2)) << 64 return (progname, progversion) def parseVer_ymd_GIT_SHASUM(match, filepath): progname = match.group(1) progversion = ( (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | (int(match.group(4)) << 32) ) return (progname, progversion) def parseVer_ymd(match, filepath): progname = match.group(1) progversion = ( (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | (int(match.group(4)) << 32) ) return (progname, progversion) def parseVer_GIT(match, filepath): progname = match.group(1) st = os.stat(filepath) progversion = int(st.st_mtime) << 64 return (progname, progversion) extensions = ( ".tar.gz", ".tar.bz2", ".tar.xz", ".orig.tar.gz", ".orig.tar.bz2", ".orig.tar.xz", ".zip", ".tgz", ".tbz", ".txz", ) versionRegex = ( (re.compile(r"(gcc[-_]\d+)\.(\d+)\.(\d+)"), parseVer_12), # gcc.1.2 (re.compile(r"(linux[-_]\d+\.\d+)\.(\d+)"), parseVer_r), # linux.1 (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)\.(\d+)"), parseVer_1234), # xxx-1.2.3.4 ( re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)-"), parseVer_ymd_GIT_SHASUM, ), # xxx-YYYY-MM-DD-GIT_SHASUM (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)"), parseVer_ymd), # xxx-YYYY-MM-DD (re.compile(r"(.+)[-_]([0-9a-fA-F]{40,40})"), parseVer_GIT), # xxx-GIT_SHASUM (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-1.2.3a (re.compile(r"(.+)[-_]v(\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-v1.2.3a (re.compile(r"(.+)[-_](\d+)_(\d+)_(\d+)"), parseVer_123), # xxx-1_2_3 (re.compile(r"(.+)[-_](\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-1.2a (re.compile(r"(.+)[-_]v(\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-v1.2a (re.compile(r"(.+)[-_]r?(\d+)"), parseVer_r), # xxx-r1111 ) blacklist = [ ("wl_apsta", re.compile(r"wl_apsta.*")), (".fw", re.compile(r".*\.fw")), (".arm", re.compile(r".*\.arm")), (".bin", re.compile(r".*\.bin")), ("rt-firmware", re.compile(r"RT[\d\w]+_Firmware.*")), ] class EntryParseError(Exception): pass class Entry: def __init__(self, directory, builddir, filename): self.directory = directory self.filename = filename self.builddir = builddir self.progname = "" self.fileext = "" self.filenoext = "" for ext in extensions: if filename.endswith(ext): filename = filename[0 : 0 - len(ext)] self.filenoext = filename self.fileext = ext break else: print(self.filename, "has an unknown file-extension") raise EntryParseError("ext") for (regex, parseVersion) in versionRegex: match = regex.match(filename) if match: (self.progname, self.version) = parseVersion( match, directory + "/" + filename + self.fileext ) break else: print(self.filename, "has an unknown version pattern") raise EntryParseError("ver") def getPath(self): return (self.directory + "/" + self.filename).replace("//", "/") def getBuildPaths(self): paths = [] for subdir in os.scandir(self.builddir): package_build_dir = os.path.join(subdir.path, self.filenoext) if os.path.exists(package_build_dir): paths.append(package_build_dir) return paths def deleteFile(self): path = self.getPath() print("Deleting", path) if not opt_dryrun: os.unlink(path) def deleteBuildDir(self): paths = self.getBuildPaths() for path in paths: print("Deleting BuildDir", path) if not opt_dryrun: shutil.rmtree(path) def __ge__(self, y): return self.version >= y.version def usage(): print("OpenWrt download directory cleanup utility") print("Usage: " + sys.argv[0] + " [OPTIONS] ") print("") print(" -d|--dry-run Do a dry-run. Don't delete any files") print(" -B|--show-blacklist Show the blacklist and exit") print(" -w|--whitelist ITEM Remove ITEM from blacklist") print( " -D|--download-dir Provide path to dl dir to clean also the build directory" ) print( " -b|--build-dir Provide path to build dir to clean also the build directory" ) def main(argv): global opt_dryrun try: (opts, args) = getopt.getopt( argv[1:], "hdBwDb:", [ "help", "dry-run", "show-blacklist", "whitelist=", "download-dir=", "build-dir=", ], ) except getopt.GetoptError as e: usage() return 1 directory = "dl/" builddir = "build_dir/" for (o, v) in opts: if o in ("-h", "--help"): usage() return 0 if o in ("-d", "--dry-run"): opt_dryrun = True if o in ("-w", "--whitelist"): for i in range(0, len(blacklist)): (name, regex) = blacklist[i] if name == v: del blacklist[i] break else: print("Whitelist error: Item", v, "is not in blacklist") return 1 if o in ("-B", "--show-blacklist"): for (name, regex) in blacklist: sep = "\t\t" if len(name) >= 8: sep = "\t" print("%s%s(%s)" % (name, sep, regex.pattern)) return 0 if o in ("-D", "--download-dir"): directory = v if o in ("-b", "--build-dir"): builddir = v if not os.path.exists(directory): print("Can't find dl path", directory) return 1 if not os.path.exists(builddir): print("Can't find dl path", builddir) return 1 # Create a directory listing and parse the file names. entries = [] for filename in os.listdir(directory): if filename == "." or filename == "..": continue for (name, regex) in blacklist: if regex.match(filename): if opt_dryrun: print(filename, "is blacklisted") break else: try: entries.append(Entry(directory, builddir, filename)) except EntryParseError as e: pass # Create a map of programs progmap = {} for entry in entries: if entry.progname in progmap.keys(): progmap[entry.progname].append(entry) else: progmap[entry.progname] = [ entry, ] # Traverse the program map and delete everything but the last version for prog in progmap: lastVersion = None versions = progmap[prog] for version in versions: if lastVersion is None or version >= lastVersion: lastVersion = version if lastVersion: for version in versions: if version is not lastVersion: version.deleteFile() if builddir: version.deleteBuildDir() if opt_dryrun: print("Keeping", lastVersion.getPath()) return 0 if __name__ == "__main__": sys.exit(main(sys.argv))