blob: ece045ee35b1f45205d7451d0f3c52b483a51098 [file] [log] [blame]
#!/bin/env python
# Breakpad symbol cleanup
# When the Breakpad symbol store gets built as part of the
# Mozilla build process, it creates a file listing the symbols
# for that particular build. The files are named:
# ${product}-${version}-${OS_ARCH}-${BUILD_ID}[-${BUILDID_EXTRA}]-symbols.txt, i.e.:
# firefox-3.0a5pre-WINNT-2007042804-symbols.txt
# $BUILDID_EXTRA is an optional identifier for feature branches, etc.
# This script, given a path to a symbol store, removes symbols
# for the oldest builds there.
import os
import os.path
import sys
import re
from datetime import datetime, timedelta
from optparse import OptionParser
# options, tweak as desired
# maximum number of nightlies to keep per branch
nightliesPerBin = 30
# maximum age permitted for a set of symbols, in days.
# used to clean up old feature branches, for example
maxNightlyAge = timedelta(45)
# end options
# RE to get the version number without alpha/beta designation
versionRE = re.compile("^(\d+\.\d+)")
parser = OptionParser(usage="usage: %prog [options] <symbol path> [symbol indexes to remove]")
parser.add_option("-d", "--dry-run",
action="store_true", dest="dry_run", default=False,
help="Don't delete anything, just print a list of actions")
parser.add_option("-r", "--remove-these-symbols",
action="store_true", dest="remove_symbols",
help="Remove specified symbol indexes and their contained symbols")
(options, args) = parser.parse_args()
if not args:
print >>sys.stderr, "Must specify a symbol path!"
symbolPath = args[0]
if (options.remove_symbols and len(args) < 2) or (not options.remove_symbols and len(args) > 1):
print >>sys.stderr, "Bad commandline"
symbols_to_remove = set()
if options.remove_symbols:
symbols_to_remove = set(os.path.basename(a) for a in args[1:])
# Cheezy atom implementation, so we don't have to store symbol filenames
# multiple times.
atoms = []
atomdict = {}
def atomize(s):
if s in atomdict:
return atomdict[s]
a = len(atoms)
atomdict[s] = a
return a
def sortByBuildID(x, y):
"Sort two symbol index filenames by the Build IDs contained within"
(a, b) = (os.path.basename(x).split('-')[3],
return cmp(a, b)
buildidRE = re.compile("(\d\d\d\d)(\d\d)(\d\d)(\d\d)")
def datetimefrombuildid(f):
"""Given a symbol index filename, return a datetime representing the
Build ID contained within it."""
m = buildidRE.match(os.path.basename(f).split('-')[3])
if m:
return datetime(*[int(x) for x in m.groups()])
# punt
def adddefault(d, key, default):
"If d[key] does not exist, set d[key] = default."
if key not in d:
d[key] = default
def addFiles(symbolindex, filesDict):
"""Return a list of atoms representing the symbols in this index file.
Also add 1 to filesDict[atom] for each symbol."""
l = []
sf = open(symbolindex, "r")
for line in sf:
a = atomize(line.rstrip())
adddefault(filesDict, a, 0)
filesDict[a] += 1
except IOError:
return l
def markDeleteSymbols(symbols, filesDict):
"Decrement reference count by one for each symbol in this symbol index."
for a in symbols:
filesDict[a] -= 1
def deletefile(f):
if options.dry_run:
print "rm ", f
if os.path.isfile(f):
except OSError:
print >>sys.stderr, "Error removing file: ", f
builds = {}
allfiles = {}
buildfiles = {}
print "[1/4] Reading symbol index files..."
# get symbol index files, there's one per build
for f in os.listdir(symbolPath):
if not (os.path.isfile(os.path.join(symbolPath, f)) and
# increment reference count of all symbol files listed in the index
# and also keep a list of files from that index
buildfiles[f] = addFiles(os.path.join(symbolPath, f), allfiles)
# drop -symbols.txt
parts = f.split("-")[:-1]
(product, version, osName, buildId) = parts[:4]
# extract branch
# nightly build versions end with "pre" (older branches)
# or "a1" (new mozilla-central) or "a2" (aurora)
if version.endswith("a1"):
branch = "nightly"
elif version.endswith("a2"):
branch = "aurora"
elif version.endswith("pre"):
m = versionRE.match(version)
if m:
branch =
branch = version
branch = "release"
# group into bins by branch-product-os[-featurebranch]
identifier = "%s-%s-%s" % (branch, product, osName)
if len(parts) > 4: # extra buildid, probably
identifier += "-" + "-".join(parts[4:])
adddefault(builds, identifier, [])
if f in symbols_to_remove:
markDeleteSymbols(buildfiles[f], allfiles)
print "[2/4] Looking for symbols to delete..."
if not symbols_to_remove:
oldestdate = - maxNightlyAge
for bin in builds:
if bin.startswith("release"):
# Skip release builds for now
if len(builds[bin]) > nightliesPerBin:
# delete the oldest builds if there are too many
for f in builds[bin][:-nightliesPerBin]:
markDeleteSymbols(buildfiles[f], allfiles)
builds[bin] = builds[bin][-nightliesPerBin:]
# now look for really old symbol files
for f in builds[bin]:
if datetimefrombuildid(f) < oldestdate:
markDeleteSymbols(buildfiles[f], allfiles)
print "[3/4] Deleting symbols..."
# now delete all files marked for deletion
for a, refcnt in allfiles.iteritems():
if refcnt == 0:
deletefile(os.path.join(symbolPath, atoms[a]))
print "[4/4] Pruning empty directories..."
# now delete empty directories.
for root, dirs, files in os.walk(symbolPath, topdown=False):
for d in dirs:
fullpath = os.path.join(root, d)
if len(os.listdir(fullpath)) == 0:
if options.dry_run:
print "rm -rf ", fullpath
print "Done!"