| #!/usr/bin/env python |
| # Written for Mozilla by Chris AtLee <catlee@mozilla.com> 2008 |
| """Delete old buildbot builds to make room for the current build. |
| |
| %prog [options] base_dir1 [base_dir2 ...] |
| |
| base_dir1 is the root of the directory tree you want to delete builds |
| from. |
| |
| Sub-directories of base_dir1 will be deleted, in order from oldest to newest, |
| until the specified amount of space is free. |
| |
| base_dir1 will always be used for space calculations, but if other base_dir# |
| are provided, subdirectories within those dirs will also be purged. This will |
| obviously only increase the available space if the other base_dirs are on the |
| same mountpoint, but this can be useful for, e.g., cleaning up scratchbox. |
| |
| example: |
| python %prog -s 6 /builds/moz2_slave /scratchbox/users/cltbld/home/cltbld/build |
| """ |
| |
| import os |
| import shutil |
| import sys |
| from fnmatch import fnmatch |
| import re |
| |
| DEFAULT_BASE_DIRS = [".."] |
| |
| clobber_suffix = '.deleteme' |
| |
| if sys.platform == 'win32': |
| # os.statvfs doesn't work on Windows |
| from win32file import RemoveDirectory, DeleteFile, \ |
| GetFileAttributesW, SetFileAttributesW, GetDiskFreeSpace, \ |
| FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY |
| from win32api import FindFiles |
| |
| def freespace(p): |
| secsPerClus, bytesPerSec, nFreeClus, totClus = GetDiskFreeSpace(p) |
| return secsPerClus * bytesPerSec * nFreeClus |
| else: |
| def freespace(p): |
| "Returns the number of bytes free under directory `p`" |
| r = os.statvfs(p) |
| return r.f_frsize * r.f_bavail |
| |
| |
| def mtime_sort(p1, p2): |
| "sorting function for sorting a list of paths by mtime" |
| return cmp(os.path.getmtime(p1), os.path.getmtime(p2)) |
| |
| |
| def rmdirRecursiveWindows(dir): |
| """Windows-specific version of rmdirRecursive that handles |
| path lengths longer than MAX_PATH. |
| """ |
| |
| dir = os.path.realpath(dir) |
| # Make sure directory is writable |
| SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL) |
| |
| for ffrec in FindFiles('\\\\?\\' + dir + '\*.*'): |
| file_attr = ffrec[0] |
| name = ffrec[8] |
| if name == '.' or name == '..': |
| continue |
| full_name = os.path.join(dir, name) |
| |
| if file_attr & FILE_ATTRIBUTE_DIRECTORY: |
| rmdirRecursiveWindows(full_name) |
| else: |
| SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL) |
| DeleteFile('\\\\?\\' + full_name) |
| RemoveDirectory('\\\\?\\' + dir) |
| |
| |
| def rmdirRecursive(dir): |
| """This is a replacement for shutil.rmtree that works better under |
| windows. Thanks to Bear at the OSAF for the code. |
| (Borrowed from buildbot.slave.commands)""" |
| if os.name == 'nt': |
| rmdirRecursiveWindows(dir) |
| return |
| |
| if not os.path.exists(dir): |
| # This handles broken links |
| if os.path.islink(dir): |
| os.remove(dir) |
| return |
| |
| if os.path.islink(dir): |
| os.remove(dir) |
| return |
| |
| # Verify the directory is read/write/execute for the current user |
| os.chmod(dir, 0700) |
| |
| for name in os.listdir(dir): |
| full_name = os.path.join(dir, name) |
| # on Windows, if we don't have write permission we can't remove |
| # the file/directory either, so turn that on |
| if os.name == 'nt': |
| if not os.access(full_name, os.W_OK): |
| # I think this is now redundant, but I don't have an NT |
| # machine to test on, so I'm going to leave it in place |
| # -warner |
| os.chmod(full_name, 0600) |
| |
| if os.path.isdir(full_name): |
| rmdirRecursive(full_name) |
| else: |
| # Don't try to chmod links |
| if not os.path.islink(full_name): |
| os.chmod(full_name, 0700) |
| os.remove(full_name) |
| os.rmdir(dir) |
| |
| |
| def str2seconds(s): |
| """ Accepts time intervals resembling: |
| 30d (30 days) |
| 10h (10 hours) |
| Returns the specified interval as a positive integer in seconds. |
| """ |
| m = re.match(r'^(\d+)([dh])$', s) |
| if (m): |
| mul = {'d': 24*60*60, 'h': 60*60} |
| n = int(m.group(1)) |
| unit = m.group(2) |
| return n * mul[unit] |
| else: |
| raise ValueError("Unhandled time format '%s'" % s) |
| |
| |
| def purge(base_dirs, gigs, ignore, max_age, dry_run=False): |
| """Delete directories under `base_dirs` until `gigs` GB are free. |
| |
| Delete any directories older than max_age. |
| |
| Will not delete directories listed in the ignore list except |
| those tagged with an expiry threshold. Example: |
| |
| rel-*:40d |
| |
| Will not delete rel-* directories until they are over 40 days old. |
| """ |
| gigs *= 1024 * 1024 * 1024 |
| |
| # convert 'ignore' to a dict resembling { directory: cutoff_time } |
| # where a cutoff time of -1 means 'never expire'. |
| ignore = dict(map(lambda x: x.split(':')[0:2] if len(x.split(':')) > 1 else [x, -1], ignore)) |
| ignore = dict(map(lambda key: [key, time.time() - str2seconds(ignore[key])] if ignore[key] != -1 else [key, ignore[key]], ignore)) |
| |
| dirs = [] |
| for base_dir in base_dirs: |
| if os.path.exists(base_dir): |
| for d in os.listdir(base_dir): |
| p = os.path.join(base_dir, d) |
| if not os.path.isdir(p): |
| continue |
| mtime = os.path.getmtime(p) |
| skip = False |
| for pattern, cutoff_time in ignore.iteritems(): |
| if (fnmatch(d, pattern)): |
| if cutoff_time == -1 or mtime > cutoff_time: |
| skip = True |
| break |
| else: |
| print("Ignored directory '%s' exceeds cutoff time" % d) |
| if skip: |
| continue |
| dirs.append((mtime, p)) |
| |
| dirs.sort() |
| |
| while dirs: |
| mtime, d = dirs.pop(0) |
| |
| # If we're newer than max_age, and don't need any more free space, |
| # we're all done here |
| if (not max_age) or (mtime > max_age): |
| if freespace(base_dirs[0]) >= gigs: |
| break |
| |
| print "Deleting", d |
| if not dry_run: |
| try: |
| clobber_path = d + clobber_suffix |
| if os.path.exists(clobber_path): |
| rmdirRecursive(clobber_path) |
| # Prevent repeated moving. |
| if d.endswith(clobber_suffix): |
| rmdirRecursive(d) |
| else: |
| shutil.move(d, clobber_path) |
| rmdirRecursive(clobber_path) |
| except: |
| print >>sys.stderr, "Couldn't purge %s properly. Skipping." % d |
| |
| |
| def purge_hg_shares(share_dir, gigs, max_age, dry_run=False): |
| """Deletes old hg directories under share_dir""" |
| # Find hg directories |
| hg_dirs = [] |
| for root, dirs, files in os.walk(share_dir): |
| for d in dirs[:]: |
| path = os.path.join(root, d, '.hg') |
| if os.path.exists(path) or os.path.exists(path + clobber_suffix): |
| hg_dirs.append(os.path.join(root, d)) |
| # Remove d from the list so we don't go traversing down into it |
| dirs.remove(d) |
| |
| # Now we have a list of hg directories, call purge on them |
| purge(hg_dirs, gigs, [], max_age, dry_run) |
| |
| # Clean up empty directories |
| for d in hg_dirs: |
| if not os.path.exists(os.path.join(d, '.hg')): |
| print "Cleaning up", d |
| if not dry_run: |
| rmdirRecursive(d) |
| |
| if __name__ == '__main__': |
| import time |
| from optparse import OptionParser |
| from ConfigParser import ConfigParser, NoOptionError |
| |
| max_age = 14 |
| config = ConfigParser() |
| config.read(os.path.expanduser('~/.purge_builds.cfg')) |
| try: |
| max_age = config.getint('DEFAULT', 'max_age') |
| except (NoOptionError, ValueError): |
| pass |
| |
| cwd = os.path.basename(os.getcwd()) |
| parser = OptionParser(usage=__doc__) |
| parser.set_defaults(size=5, share_size=1, skip=[cwd], dry_run=False, max_age=max_age) |
| |
| parser.add_option('-s', '--size', |
| help='free space required (in GB, default 5)', dest='size', |
| type='float') |
| |
| parser.add_option('--share-size', |
| help='free space required for vcs shares (in GB, default 1)', dest='share_size', |
| type='float') |
| |
| parser.add_option('-n', '--not', help='do not delete this directory. Append :30d to skip for up to 30 days, or :30h to skip for up to 30 hours', |
| action='append', dest='skip') |
| |
| parser.add_option('', '--dry-run', action='store_true', |
| dest='dry_run', |
| help='''do not delete anything, just print out what would be |
| deleted. note that since no directories are deleted, if the amount of free |
| disk space in base_dir(s) is less than the required size, then ALL directories |
| will be listed in the order in which they would be deleted.''') |
| |
| parser.add_option('', '--max-age', dest='max_age', type='int', |
| help='''maximum age (in days) for directories. If any directory |
| has an mtime older than this, it will be deleted, regardless of how |
| much free space is required. Set to 0 to disable.''') |
| |
| options, base_dirs = parser.parse_args() |
| |
| if len(base_dirs) < 1: |
| for d in DEFAULT_BASE_DIRS: |
| if os.path.exists(d): |
| base_dirs.append(d) |
| if len(base_dirs) < 1: |
| parser.error("Must specify one or more base_dirs") |
| sys.exit(1) |
| |
| # Figure out the mtime before which we'll start deleting old directories |
| if options.max_age: |
| cutoff_time = time.time() - 24 * 3600 * options.max_age |
| else: |
| cutoff_time = None |
| |
| purge(base_dirs, options.size, options.skip, cutoff_time, options.dry_run) |
| |
| # Try to cleanup shared hg repos. We run here even if we've freed enough |
| # space so we can be sure and delete repositories older than max_age |
| if 'HG_SHARE_BASE_DIR' in os.environ: |
| purge_hg_shares(os.environ['HG_SHARE_BASE_DIR'], |
| options.share_size, cutoff_time, options.dry_run) |
| |
| # tooltool cache cleanup |
| if 'TOOLTOOL_HOME' in os.environ and 'TOOLTOOL_CACHE' in os.environ: |
| import imp |
| try: |
| tooltool = imp.load_source('tooltool', os.path.join(os.environ['TOOLTOOL_HOME'], "tooltool.py")) |
| tooltool.purge(os.environ['TOOLTOOL_CACHE'], options.size) |
| except: |
| print "Warning: impossible to cleanup tooltool cache" |
| |
| after = freespace(base_dirs[0]) / (1024 * 1024 * 1024.0) |
| |
| # Try to cleanup the current dir if we still need space and it will |
| # actually help. |
| if after < options.size: |
| # We skip the tools dir here because we've usually just cloned it. |
| purge(['.'], options.size, ['tools'], cutoff_time, options.dry_run) |
| after = freespace(base_dirs[0]) / (1024 * 1024 * 1024.0) |
| |
| if after < options.size: |
| print "Error: unable to free %1.2f GB of space. " % options.size + \ |
| "Free space only %1.2f GB" % after |
| sys.exit(1) |
| else: |
| print "%1.2f GB of space available" % after |
| sys.exit(0) |
