| #!/usr/bin/env python |
| |
| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this file, |
| # You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| """ |
| Setup mozbase packages for development. |
| |
| Packages may be specified as command line arguments. |
| If no arguments are given, install all packages. |
| |
| See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase |
| """ |
| |
| import os |
| import subprocess |
| import sys |
| from optparse import OptionParser |
| from subprocess import PIPE |
| try: |
| from subprocess import check_call as call |
| except ImportError: |
| from subprocess import call |
| |
| |
| # directory containing this file |
| here = os.path.dirname(os.path.abspath(__file__)) |
| |
| # all python packages |
| mozbase_packages = [i for i in os.listdir(here) |
| if os.path.exists(os.path.join(here, i, 'setup.py'))] |
| test_packages = [ "mock" # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests |
| ] |
| extra_packages = [ "sphinx" # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation |
| ] |
| |
| def cycle_check(order, dependencies): |
| """ensure no cyclic dependencies""" |
| order_dict = dict([(j, i) for i, j in enumerate(order)]) |
| for package, deps in dependencies.items(): |
| index = order_dict[package] |
| for d in deps: |
| assert index > order_dict[d], "Cyclic dependencies detected" |
| |
| def info(directory): |
| "get the package setup.py information" |
| |
| assert os.path.exists(os.path.join(directory, 'setup.py')) |
| |
| # setup the egg info |
| try: |
| call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE) |
| except subprocess.CalledProcessError: |
| print "Error running setup.py in %s" % directory |
| raise |
| |
| # get the .egg-info directory |
| egg_info = [entry for entry in os.listdir(directory) |
| if entry.endswith('.egg-info')] |
| assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info) |
| egg_info = os.path.join(directory, egg_info[0]) |
| assert os.path.isdir(egg_info), "%s is not a directory" % egg_info |
| |
| # read the package information |
| pkg_info = os.path.join(egg_info, 'PKG-INFO') |
| info_dict = {} |
| for line in file(pkg_info).readlines(): |
| if not line or line[0].isspace(): |
| continue # XXX neglects description |
| assert ':' in line |
| key, value = [i.strip() for i in line.split(':', 1)] |
| info_dict[key] = value |
| |
| return info_dict |
| |
| def get_dependencies(directory): |
| "returns the package name and dependencies given a package directory" |
| |
| # get the package metadata |
| info_dict = info(directory) |
| |
| # get the .egg-info directory |
| egg_info = [entry for entry in os.listdir(directory) |
| if entry.endswith('.egg-info')][0] |
| |
| # read the dependencies |
| requires = os.path.join(directory, egg_info, 'requires.txt') |
| dependencies = [] |
| if os.path.exists(requires): |
| for line in file(requires): |
| line = line.strip() |
| # in requires.txt file, a dependency is a non empty line |
| # Also lines like [device] are sections to mark optional |
| # dependencies, we don't want those sections. |
| if line and not (line.startswith('[') and line.endswith(']')): |
| dependencies.append(line) |
| |
| # return the information |
| return info_dict['Name'], dependencies |
| |
| def dependency_info(dep): |
| "return dictionary of dependency information from a dependency string" |
| retval = dict(Name=None, Type=None, Version=None) |
| for joiner in ('==', '<=', '>='): |
| if joiner in dep: |
| retval['Type'] = joiner |
| name, version = [i.strip() for i in dep.split(joiner, 1)] |
| retval['Name'] = name |
| retval['Version'] = version |
| break |
| else: |
| retval['Name'] = dep.strip() |
| return retval |
| |
| def unroll_dependencies(dependencies): |
| """ |
| unroll a set of dependencies to a flat list |
| |
| dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']), |
| 'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']), |
| 'packageC': set(['packageE']), |
| 'packageE': set(['packageF', 'packageG']), |
| 'packageF': set(['packageG']), |
| 'packageX': set(['packageA', 'packageG'])} |
| """ |
| |
| order = [] |
| |
| # flatten all |
| packages = set(dependencies.keys()) |
| for deps in dependencies.values(): |
| packages.update(deps) |
| |
| while len(order) != len(packages): |
| |
| for package in packages.difference(order): |
| if set(dependencies.get(package, set())).issubset(order): |
| order.append(package) |
| break |
| else: |
| raise AssertionError("Cyclic dependencies detected") |
| |
| cycle_check(order, dependencies) # sanity check |
| |
| return order |
| |
| |
| def main(args=sys.argv[1:]): |
| |
| # parse command line options |
| usage = '%prog [options] [package] [package] [...]' |
| parser = OptionParser(usage=usage, description=__doc__) |
| parser.add_option('-d', '--dependencies', dest='list_dependencies', |
| action='store_true', default=False, |
| help="list dependencies for the packages") |
| parser.add_option('--list', action='store_true', default=False, |
| help="list what will be installed") |
| parser.add_option('--extra', '--install-extra-packages', action='store_true', default=False, |
| help="installs extra supporting packages as well as core mozbase ones") |
| options, packages = parser.parse_args(args) |
| |
| if not packages: |
| # install all packages |
| packages = sorted(mozbase_packages) |
| |
| # ensure specified packages are in the list |
| assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages) |
| |
| if options.list_dependencies: |
| # list the package dependencies |
| for package in packages: |
| print '%s: %s' % get_dependencies(os.path.join(here, package)) |
| parser.exit() |
| |
| # gather dependencies |
| # TODO: version conflict checking |
| deps = {} |
| alldeps = {} |
| mapping = {} # mapping from subdir name to package name |
| # core dependencies |
| for package in packages: |
| key, value = get_dependencies(os.path.join(here, package)) |
| deps[key] = [dependency_info(dep)['Name'] for dep in value] |
| mapping[package] = key |
| |
| # keep track of all dependencies for non-mozbase packages |
| for dep in value: |
| alldeps[dependency_info(dep)['Name']] = ''.join(dep.split()) |
| |
| # indirect dependencies |
| flag = True |
| while flag: |
| flag = False |
| for value in deps.values(): |
| for dep in value: |
| if dep in mozbase_packages and dep not in deps: |
| key, value = get_dependencies(os.path.join(here, dep)) |
| deps[key] = [dep for dep in value] |
| |
| for dep in value: |
| alldeps[dep] = ''.join(dep.split()) |
| mapping[package] = key |
| flag = True |
| break |
| if flag: |
| break |
| |
| # get the remaining names for the mapping |
| for package in mozbase_packages: |
| if package in mapping: |
| continue |
| key, value = get_dependencies(os.path.join(here, package)) |
| mapping[package] = key |
| |
| # unroll dependencies |
| unrolled = unroll_dependencies(deps) |
| |
| # make a reverse mapping: package name -> subdirectory |
| reverse_mapping = dict([(j,i) for i, j in mapping.items()]) |
| |
| # we only care about dependencies in mozbase |
| unrolled = [package for package in unrolled if package in reverse_mapping] |
| |
| if options.list: |
| # list what will be installed |
| for package in unrolled: |
| print package |
| parser.exit() |
| |
| # set up the packages for development |
| for package in unrolled: |
| call([sys.executable, 'setup.py', 'develop', '--no-deps'], |
| cwd=os.path.join(here, reverse_mapping[package])) |
| |
| # add the directory of sys.executable to path to aid the correct |
| # `easy_install` getting called |
| # https://bugzilla.mozilla.org/show_bug.cgi?id=893878 |
| os.environ['PATH'] = '%s%s%s' % (os.path.dirname(os.path.abspath(sys.executable)), |
| os.path.pathsep, |
| os.environ.get('PATH', '').strip(os.path.pathsep)) |
| |
| # install non-mozbase dependencies |
| # these need to be installed separately and the --no-deps flag |
| # subsequently used due to a bug in setuptools; see |
| # https://bugzilla.mozilla.org/show_bug.cgi?id=759836 |
| pypi_deps = dict([(i, j) for i,j in alldeps.items() |
| if i not in unrolled]) |
| for package, version in pypi_deps.items(): |
| # easy_install should be available since we rely on setuptools |
| call(['easy_install', version]) |
| |
| # install packages required for unit testing |
| for package in test_packages: |
| call(['easy_install', package]) |
| |
| # install extra non-mozbase packages if desired |
| if options.extra: |
| for package in extra_packages: |
| call(['easy_install', package]) |
| |
| if __name__ == '__main__': |
| main() |
