| #!/usr/bin/env python |
| |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| """ |
| Given a list of packages and the versions to mirror, |
| generate a diff appropriate for mirroring |
| https://github.com/mozilla/mozbase |
| to http://hg.mozilla.org/mozilla-central/file/tip/testing/mozbase |
| |
| If a package version is not given, the latest version will be used. |
| |
| Note that this shells out to `cp` for simplicity, so you should run this |
| somewhere that has the `cp` command available. |
| |
| Your mozilla-central repository must have no outstanding changes before this |
| script is run. The repository must also have no untracked |
| files that show up in `hg st`. |
| |
| See: https://bugzilla.mozilla.org/show_bug.cgi?id=702832 |
| """ |
| |
| import imp |
| import optparse |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| |
| from pkg_resources import parse_version |
| from subprocess import check_call as call |
| |
| # globals |
| here = os.path.dirname(os.path.abspath(__file__)) |
| MOZBASE = 'git://github.com/mozilla/mozbase.git' |
| version_regex = r"""PACKAGE_VERSION *= *['"]([0-9.]+)["'].*""" |
| setup_development = imp.load_source('setup_development', |
| os.path.join(here, 'setup_development.py')) |
| current_package = None |
| current_package_info = {} |
| |
| def error(msg): |
| """err out with a message""" |
| print >> sys.stdout, msg |
| sys.exit(1) |
| |
| def remove(path): |
| """remove a file or directory""" |
| if os.path.isdir(path): |
| shutil.rmtree(path) |
| else: |
| os.remove(path) |
| |
| ### git functions |
| |
| def latest_commit(git_dir): |
| """returns last commit hash from a git repository directory""" |
| command = ['git', 'log', '--pretty=format:%H', 'HEAD^..HEAD'] |
| process = subprocess.Popen(command, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| cwd=git_dir) |
| stdout, stderr = process.communicate() |
| return stdout.strip() |
| |
| def tags(git_dir): |
| """return all tags in a git repository""" |
| |
| command = ['git', 'tag'] |
| process = subprocess.Popen(command, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| cwd=git_dir) |
| stdout, stderr = process.communicate() |
| return [line.strip() for line in stdout.strip().splitlines()] |
| |
| def checkout(git_dir, tag): |
| """checkout a tagged version of a git repository""" |
| |
| command = ['git', 'checkout', tag] |
| process = subprocess.Popen(command, |
| cwd=git_dir) |
| process.communicate() |
| |
| |
| ### hg functions |
| |
| def untracked_files(hg_dir): |
| """untracked files in an hg repository""" |
| process = subprocess.Popen(['hg', 'st'], |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| cwd=hg_dir) |
| stdout, stderr = process.communicate() |
| lines = [line.strip() for line in stdout.strip().splitlines()] |
| status = [line.split(None, 1) for line in lines] |
| return [j for i, j in status if i == '?'] |
| |
| def revert(hg_dir, excludes=()): |
| """revert a hg repository directory""" |
| call(['hg', 'revert', '--no-backup', '--all'], cwd=hg_dir) |
| newfiles = untracked_files(hg_dir) |
| for f in newfiles: |
| path = os.path.join(hg_dir, f) |
| if path not in excludes: |
| os.remove(path) |
| |
| ### |
| |
| def generate_packages_txt(): |
| """ |
| generate a packages.txt file appropriate for |
| http://mxr.mozilla.org/mozilla-central/source/build/virtualenv/populate_virtualenv.py |
| |
| See also: |
| http://mxr.mozilla.org/mozilla-central/source/build/virtualenv/packages.txt |
| """ |
| |
| prefix = 'testing/mozbase/' # relative path from topsrcdir |
| |
| # gather the packages |
| packages = setup_development.mozbase_packages |
| |
| # write them in the appropriate format |
| path = os.path.join(here, 'packages.txt') |
| with file(path, 'w') as f: |
| for package in sorted(packages): |
| f.write("%s.pth:%s%s\n" % (package, prefix, package)) |
| |
| ### version-related functions |
| |
| def parse_versions(*args): |
| """return a list of 2-tuples of (directory, version)""" |
| |
| retval = [] |
| for arg in args: |
| if '=' in arg: |
| directory, version = arg.split('=', 1) |
| else: |
| directory = arg |
| version = None |
| retval.append((directory, version)) |
| return retval |
| |
| def version_tag(directory, version): |
| """return a version tag string given the directory name of the package""" |
| package = current_package_info[directory]['name'] |
| return '%s-%s' % (package, version) |
| |
| def setup(**kwargs): |
| """monkey-patch function for setuptools.setup""" |
| assert current_package |
| current_package_info[current_package] = kwargs |
| |
| def checkout_tag(src, directory, version): |
| """ |
| front end to checkout + version_tag; |
| if version is None, checkout HEAD |
| """ |
| |
| if version is None: |
| tag = 'master' |
| else: |
| tag = version_tag(directory, version) |
| checkout(src, tag) |
| |
| def check_consistency(*package_info): |
| """checks consistency between a set of packages""" |
| |
| # set versions and dependencies per package |
| versions = {} |
| dependencies = {} |
| for package in package_info: |
| name = package['name'] |
| versions[name] = package['version'] |
| for dep in package.get('install_requires', []): |
| dependencies.setdefault(name, []).append(dep) |
| |
| func_map = {'==': tuple.__eq__, |
| '<=': tuple.__le__, |
| '>=': tuple.__ge__} |
| |
| # check dependencies |
| errors = [] |
| for package, deps in dependencies.items(): |
| for dep in deps: |
| parsed = setup_development.dependency_info(dep) |
| if parsed['Name'] not in versions: |
| # external dependency |
| continue |
| if parsed.get('Version') is None: |
| # no version specified for dependency |
| continue |
| |
| # check versions |
| func = func_map[parsed['Type']] |
| comparison = func(parse_version(versions[parsed['Name']]), |
| parse_version(parsed['Version'])) |
| |
| if not comparison: |
| # an error |
| errors.append("Dependency for package '%s' failed: %s-%s not %s %s" % (package, parsed['Name'], versions[parsed['Name']], parsed['Type'], parsed['Version'])) |
| |
| # raise an Exception if errors exist |
| if errors: |
| raise Exception('\n'.join(errors)) |
| |
| ### |
| |
| def main(args=sys.argv[1:]): |
| """command line entry point""" |
| |
| # parse command line options |
| usage = '%prog [options] package1[=version1] <package2=version2> <...>' |
| class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
| """description formatter for console script entry point""" |
| def format_description(self, description): |
| if description: |
| return description.strip() + '\n' |
| else: |
| return '' |
| parser = optparse.OptionParser(usage=usage, |
| description=__doc__, |
| formatter=PlainDescriptionFormatter()) |
| parser.add_option('-o', '--output', dest='output', |
| help="specify the output file; otherwise will be in the current directory with a name based on the hash") |
| parser.add_option('--develop', dest='develop', |
| action='store_true', default=False, |
| help="use development (master) version of packages") |
| parser.add_option('--no-check', dest='check', |
| action='store_false', default=True, |
| help="Do not check current repository state") |
| parser.add_option('--packages', dest='output_packages', |
| default=False, action='store_true', |
| help="generate packages.txt and exit") |
| options, args = parser.parse_args(args) |
| if options.output_packages: |
| generate_packages_txt() |
| parser.exit() |
| if args: |
| versions = parse_versions(*args) |
| else: |
| parser.print_help() |
| parser.exit() |
| output = options.output |
| |
| # gather info from current mozbase packages |
| global current_package |
| setuptools = sys.modules.get('setuptools') |
| sys.modules['setuptools'] = sys.modules[__name__] |
| try: |
| for package in setup_development.mozbase_packages: |
| current_package = package |
| imp.load_source('setup', os.path.join(here, package, 'setup.py')) |
| finally: |
| current_package = None |
| sys.modules.pop('setuptools') |
| if setuptools: |
| sys.modules['setuptools'] = setuptools |
| assert set(current_package_info.keys()) == set(setup_development.mozbase_packages) |
| |
| # check consistency of current set of packages |
| check_consistency(*current_package_info.values()) |
| |
| # calculate hg root |
| hg_root = os.path.dirname(os.path.dirname(here)) # testing/mozbase |
| hg_dir = os.path.join(hg_root, '.hg') |
| assert os.path.exists(hg_dir) and os.path.isdir(hg_dir) |
| |
| # ensure there are no outstanding changes to m-c |
| process = subprocess.Popen(['hg', 'diff'], cwd=here, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| stdout, stderr = process.communicate() |
| if stdout.strip() and options.check: |
| error("Outstanding changes in %s; aborting" % hg_root) |
| |
| # ensure that there are no untracked files in testing/mozbase |
| untracked = untracked_files(hg_root) |
| if untracked and options.check: |
| error("Untracked files in %s:\n %s\naborting" % (hg_root, '\n'.join([' %s' % i for i in untracked]))) |
| |
| tempdir = tempfile.mkdtemp() |
| try: |
| |
| # download mozbase |
| call(['git', 'clone', MOZBASE], cwd=tempdir) |
| src = os.path.join(tempdir, 'mozbase') |
| assert os.path.isdir(src) |
| if output is None: |
| commit_hash = latest_commit(src) |
| output = os.path.join(os.getcwd(), '%s.diff' % commit_hash) |
| |
| # get the tags |
| _tags = tags(src) |
| |
| # ensure all directories and tags are available |
| for index, (directory, version) in enumerate(versions): |
| |
| setup_py = os.path.join(src, directory, 'setup.py') |
| assert os.path.exists(setup_py), "'%s' not found" % setup_py |
| |
| if not version: |
| |
| if options.develop: |
| # use master of package; keep version=None |
| continue |
| |
| # choose maximum version from setup.py |
| with file(setup_py) as f: |
| for line in f.readlines(): |
| line = line.strip() |
| match = re.match(version_regex, line) |
| if match: |
| version = match.groups()[0] |
| versions[index] = (directory, version) |
| print "Using %s=%s" % (directory, version) |
| break |
| else: |
| error("Cannot find PACKAGE_VERSION in %s" % setup_py) |
| |
| tag = version_tag(directory, version) |
| if tag not in _tags: |
| error("Tag for '%s' -- %s -- not in tags:\n%s" % (directory, version, '\n'.join(sorted(_tags)))) |
| |
| # ensure that the versions to mirror are compatible with what is in m-c |
| old_package_info = current_package_info.copy() |
| setuptools = sys.modules.get('setuptools') |
| sys.modules['setuptools'] = sys.modules[__name__] |
| try: |
| for directory, version in versions: |
| |
| # checkout appropriate revision of mozbase |
| checkout_tag(src, directory, version) |
| |
| # update the package information |
| setup_py = os.path.join(src, directory, 'setup.py') |
| current_package = directory |
| imp.load_source('setup', setup_py) |
| finally: |
| current_package = None |
| sys.modules.pop('setuptools') |
| if setuptools: |
| sys.modules['setuptools'] = setuptools |
| checkout(src, 'master') |
| check_consistency(*current_package_info.values()) |
| |
| # copy mozbase directories to m-c |
| for directory, version in versions: |
| |
| # checkout appropriate revision of mozbase |
| checkout_tag(src, directory, version) |
| |
| # replace the directory |
| remove(os.path.join(here, directory)) |
| call(['cp', '-r', directory, here], cwd=src) |
| |
| # regenerate mozbase's packages.txt |
| generate_packages_txt() |
| |
| # generate the diff and write to output file |
| command = ['hg', 'addremove'] |
| # TODO: don't add untracked files via `hg addremove --exclude...` |
| call(command, cwd=hg_root) |
| process = subprocess.Popen(['hg', 'diff'], |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| cwd=hg_root) |
| stdout, stderr = process.communicate() |
| with file(output, 'w') as f: |
| f.write(stdout) |
| f.close() |
| |
| # ensure that the diff you just wrote isn't deleted |
| untracked.append(os.path.abspath(output)) |
| |
| finally: |
| # cleanup |
| if options.check: |
| revert(hg_root, untracked) |
| shutil.rmtree(tempdir) |
| |
| print "Diff at %s" % output |
| |
| if __name__ == '__main__': |
| main() |