| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| import os.path |
| import os |
| from ConfigParser import ConfigParser, NoSectionError, NoOptionError |
| from urlparse import urlparse, urljoin |
| from urllib import pathname2url, url2pathname |
| from urllib2 import urlopen |
| from collections import defaultdict |
| from compare_locales import util |
| |
| |
| class L10nConfigParser(object): |
| '''Helper class to gather application information from ini files. |
| |
| This class is working on synchronous open to read files or web data. |
| Subclass this and overwrite loadConfigs and addChild if you need async. |
| ''' |
| def __init__(self, inipath, **kwargs): |
| """Constructor for L10nConfigParsers |
| |
| inipath -- l10n.ini path |
| Optional keyword arguments are fowarded to the inner ConfigParser as |
| defaults. |
| """ |
| if os.path.isabs(inipath): |
| self.inipath = 'file:%s' % pathname2url(inipath) |
| else: |
| pwdurl = 'file:%s/' % pathname2url(os.getcwd()) |
| self.inipath = urljoin(pwdurl, inipath) |
| # l10n.ini files can import other l10n.ini files, store the |
| # corresponding L10nConfigParsers |
| self.children = [] |
| # we really only care about the l10n directories described in l10n.ini |
| self.dirs = [] |
| # optional defaults to be passed to the inner ConfigParser (unused?) |
| self.defaults = kwargs |
| |
| def getDepth(self, cp): |
| '''Get the depth for the comparison from the parsed l10n.ini. |
| |
| Overloadable to get the source depth for fennec and friends. |
| ''' |
| try: |
| depth = cp.get('general', 'depth') |
| except: |
| depth = '.' |
| return depth |
| |
| def getFilters(self): |
| '''Get the test functions from this ConfigParser and all children. |
| |
| Only works with synchronous loads, used by compare-locales, which |
| is local anyway. |
| ''' |
| filterurl = urljoin(self.inipath, 'filter.py') |
| try: |
| l = {} |
| execfile(url2pathname(urlparse(filterurl).path), {}, l) |
| if 'test' in l and callable(l['test']): |
| filters = [l['test']] |
| else: |
| filters = [] |
| except: |
| filters = [] |
| |
| for c in self.children: |
| filters += c.getFilters() |
| |
| return filters |
| |
| def loadConfigs(self): |
| """Entry point to load the l10n.ini file this Parser refers to. |
| |
| This implementation uses synchronous loads, subclasses might overload |
| this behaviour. If you do, make sure to pass a file-like object |
| to onLoadConfig. |
| """ |
| self.onLoadConfig(urlopen(self.inipath)) |
| |
| def onLoadConfig(self, inifile): |
| """Parse a file-like object for the loaded l10n.ini file.""" |
| cp = ConfigParser(self.defaults) |
| cp.readfp(inifile) |
| depth = self.getDepth(cp) |
| self.baseurl = urljoin(self.inipath, depth) |
| # create child loaders for any other l10n.ini files to be included |
| try: |
| for title, path in cp.items('includes'): |
| # skip default items |
| if title in self.defaults: |
| continue |
| # add child config parser |
| self.addChild(title, path, cp) |
| except NoSectionError: |
| pass |
| # try to load the "dirs" defined in the "compare" section |
| try: |
| self.dirs.extend(cp.get('compare', 'dirs').split()) |
| except (NoOptionError, NoSectionError): |
| pass |
| # try getting a top level compare dir, as used for fennec |
| try: |
| self.tld = cp.get('compare', 'tld') |
| # remove tld from comparison dirs |
| if self.tld in self.dirs: |
| self.dirs.remove(self.tld) |
| except (NoOptionError, NoSectionError): |
| self.tld = None |
| # try to set "all_path" and "all_url" |
| try: |
| self.all_path = cp.get('general', 'all') |
| self.all_url = urljoin(self.baseurl, self.all_path) |
| except (NoOptionError, NoSectionError): |
| self.all_path = None |
| self.all_url = None |
| return cp |
| |
| def addChild(self, title, path, orig_cp): |
| """Create a child L10nConfigParser and load it. |
| |
| title -- indicates the module's name |
| path -- indicates the path to the module's l10n.ini file |
| orig_cp -- the configuration parser of this l10n.ini |
| """ |
| cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults) |
| cp.loadConfigs() |
| self.children.append(cp) |
| |
| def getTLDPathsTuple(self, basepath): |
| """Given the basepath, return the path fragments to be used for |
| self.tld. For build runs, this is (basepath, self.tld), for |
| source runs, just (basepath,). |
| |
| @see overwritten method in SourceTreeConfigParser. |
| """ |
| return (basepath, self.tld) |
| |
| def dirsIter(self): |
| """Iterate over all dirs and our base path for this l10n.ini""" |
| url = urlparse(self.baseurl) |
| basepath = url2pathname(url.path) |
| if self.tld is not None: |
| yield self.tld, self.getTLDPathsTuple(basepath) |
| for dir in self.dirs: |
| yield dir, (basepath, dir) |
| |
| def directories(self): |
| """Iterate over all dirs and base paths for this l10n.ini as well |
| as the included ones. |
| """ |
| for t in self.dirsIter(): |
| yield t |
| for child in self.children: |
| for t in child.directories(): |
| yield t |
| |
| def allLocales(self): |
| """Return a list of all the locales of this project""" |
| return util.parseLocales(urlopen(self.all_url).read()) |
| |
| |
| class SourceTreeConfigParser(L10nConfigParser): |
| '''Subclassing L10nConfigParser to work with just the repos |
| checked out next to each other instead of intermingled like |
| we do for real builds. |
| ''' |
| |
| def __init__(self, inipath, basepath): |
| '''Add additional arguments basepath. |
| |
| basepath is used to resolve local paths via branchnames. |
| ''' |
| L10nConfigParser.__init__(self, inipath) |
| self.basepath = basepath |
| self.tld = None |
| |
| def getDepth(self, cp): |
| '''Get the depth for the comparison from the parsed l10n.ini. |
| |
| Overloaded to get the source depth for fennec and friends. |
| ''' |
| try: |
| depth = cp.get('general', 'source-depth') |
| except: |
| try: |
| depth = cp.get('general', 'depth') |
| except: |
| depth = '.' |
| return depth |
| |
| def addChild(self, title, path, orig_cp): |
| # check if there's a section with details for this include |
| # we might have to check a different repo, or even VCS |
| # for example, projects like "mail" indicate in |
| # an "include_" section where to find the l10n.ini for "toolkit" |
| details = 'include_' + title |
| if orig_cp.has_section(details): |
| branch = orig_cp.get(details, 'mozilla') |
| inipath = orig_cp.get(details, 'l10n.ini') |
| path = self.basepath + '/' + branch + '/' + inipath |
| else: |
| path = urljoin(self.baseurl, path) |
| cp = SourceTreeConfigParser(path, self.basepath, **self.defaults) |
| cp.loadConfigs() |
| self.children.append(cp) |
| |
| def getTLDPathsTuple(self, basepath): |
| """Overwrite L10nConfigParser's getTLDPathsTuple to just return |
| the basepath. |
| """ |
| return (basepath, ) |
| |
| |
| class File(object): |
| |
| def __init__(self, fullpath, file, module=None, locale=None): |
| self.fullpath = fullpath |
| self.file = file |
| self.module = module |
| self.locale = locale |
| pass |
| |
| def getContents(self): |
| # open with universal line ending support and read |
| return open(self.fullpath, 'rU').read() |
| |
| def __hash__(self): |
| f = self.file |
| if self.module: |
| f = self.module + '/' + f |
| return hash(f) |
| |
| def __str__(self): |
| return self.fullpath |
| |
| def __cmp__(self, other): |
| if not isinstance(other, File): |
| raise NotImplementedError |
| rv = cmp(self.module, other.module) |
| if rv != 0: |
| return rv |
| return cmp(self.file, other.file) |
| |
| |
| class EnumerateDir(object): |
| ignore_dirs = ['CVS', '.svn', '.hg', '.git'] |
| |
| def __init__(self, basepath, module='', locale=None, ignore_subdirs=[]): |
| self.basepath = basepath |
| self.module = module |
| self.locale = locale |
| self.ignore_subdirs = ignore_subdirs |
| pass |
| |
| def cloneFile(self, other): |
| ''' |
| Return a File object that this enumerator would return, if it had it. |
| ''' |
| return File(os.path.join(self.basepath, other.file), other.file, |
| self.module, self.locale) |
| |
| def __iter__(self): |
| # our local dirs are given as a tuple of path segments, starting off |
| # with an empty sequence for the basepath. |
| dirs = [()] |
| while dirs: |
| dir = dirs.pop(0) |
| fulldir = os.path.join(self.basepath, *dir) |
| try: |
| entries = os.listdir(fulldir) |
| except OSError: |
| # we probably just started off in a non-existing dir, ignore |
| continue |
| entries.sort() |
| for entry in entries: |
| leaf = os.path.join(fulldir, entry) |
| if os.path.isdir(leaf): |
| if entry not in self.ignore_dirs and \ |
| leaf not in [os.path.join(self.basepath, d) |
| for d in self.ignore_subdirs]: |
| dirs.append(dir + (entry,)) |
| continue |
| yield File(leaf, '/'.join(dir + (entry,)), |
| self.module, self.locale) |
| |
| |
| class LocalesWrap(object): |
| |
| def __init__(self, base, module, locales, ignore_subdirs=[]): |
| self.base = base |
| self.module = module |
| self.locales = locales |
| self.ignore_subdirs = ignore_subdirs |
| |
| def __iter__(self): |
| for locale in self.locales: |
| path = os.path.join(self.base, locale, self.module) |
| yield (locale, EnumerateDir(path, self.module, locale, |
| self.ignore_subdirs)) |
| |
| |
| class EnumerateApp(object): |
| reference = 'en-US' |
| |
| def __init__(self, inipath, l10nbase, locales=None): |
| self.setupConfigParser(inipath) |
| self.modules = defaultdict(dict) |
| self.l10nbase = os.path.abspath(l10nbase) |
| self.filters = [] |
| drive, tail = os.path.splitdrive(inipath) |
| self.addFilters(*self.config.getFilters()) |
| self.locales = locales or self.config.allLocales() |
| self.locales.sort() |
| |
| def setupConfigParser(self, inipath): |
| self.config = L10nConfigParser(inipath) |
| self.config.loadConfigs() |
| |
| def addFilters(self, *args): |
| self.filters += args |
| |
| value_map = {None: None, 'error': 0, 'ignore': 1, 'report': 2} |
| |
| def filter(self, l10n_file, entity=None): |
| '''Go through all added filters, and, |
| - map "error" -> 0, "ignore" -> 1, "report" -> 2 |
| - if filter.test returns a bool, map that to |
| False -> "ignore" (1), True -> "error" (0) |
| - take the max of all reported |
| ''' |
| rv = 0 |
| for f in reversed(self.filters): |
| try: |
| _r = f(l10n_file.module, l10n_file.file, entity) |
| except: |
| # XXX error handling |
| continue |
| if isinstance(_r, bool): |
| _r = [1, 0][_r] |
| else: |
| # map string return value to int, default to 'error', |
| # None is None |
| _r = self.value_map.get(_r, 0) |
| if _r is not None: |
| rv = max(rv, _r) |
| return ['error', 'ignore', 'report'][rv] |
| |
| def __iter__(self): |
| ''' |
| Iterate over all modules, return en-US directory enumerator, and an |
| iterator over all locales in each iteration. Per locale, the locale |
| code and an directory enumerator will be given. |
| ''' |
| dirmap = dict(self.config.directories()) |
| mods = dirmap.keys() |
| mods.sort() |
| for mod in mods: |
| if self.reference == 'en-US': |
| base = os.path.join(*(dirmap[mod] + ('locales', 'en-US'))) |
| else: |
| base = os.path.join(self.l10nbase, self.reference, mod) |
| yield (mod, EnumerateDir(base, mod, self.reference), |
| LocalesWrap(self.l10nbase, mod, self.locales, |
| [m[len(mod)+1:] for m in mods if m.startswith(mod+'/')])) |
| |
| |
| class EnumerateSourceTreeApp(EnumerateApp): |
| '''Subclass EnumerateApp to work on side-by-side checked out |
| repos, and to no pay attention to how the source would actually |
| be checked out for building. |
| |
| It's supporting applications like Fennec, too, which have |
| 'locales/en-US/...' in their root dir, but claim to be 'mobile'. |
| ''' |
| |
| def __init__(self, inipath, basepath, l10nbase, locales=None): |
| self.basepath = basepath |
| EnumerateApp.__init__(self, inipath, l10nbase, locales) |
| |
| def setupConfigParser(self, inipath): |
| self.config = SourceTreeConfigParser(inipath, self.basepath) |
| self.config.loadConfigs() |
| |
| |
| def get_base_path(mod, loc): |
| 'statics for path patterns and conversion' |
| __l10n = 'l10n/%(loc)s/%(mod)s' |
| __en_US = 'mozilla/%(mod)s/locales/en-US' |
| if loc == 'en-US': |
| return __en_US % {'mod': mod} |
| return __l10n % {'mod': mod, 'loc': loc} |
| |
| |
| def get_path(mod, loc, leaf): |
| return get_base_path(mod, loc) + '/' + leaf |
