| # This Source Code Form is subject to the terms of the Mozilla Public |
| # License, v. 2.0. If a copy of the MPL was not distributed with this |
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| |
| '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to |
| a given command line, and executes that command line with the expanded |
| arguments. |
| |
| With the --extract argument (useful for e.g. $(AR)), it extracts object files |
| from static libraries (or use those listed in library descriptors directly). |
| |
| With the --uselist argument (useful for e.g. $(CC)), it replaces all object |
| files with a list file. This can be used to avoid limitations in the length |
| of a command line. The kind of list file format used depends on the |
| EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list) |
| or 'linkerscript' for GNU ld linker scripts. |
| See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details. |
| |
| With the --symbol-order argument, followed by a file name, it will add the |
| relevant linker options to change the order in which the linker puts the |
| symbols appear in the resulting binary. Only works for ELF targets. |
| ''' |
| from __future__ import with_statement |
| import sys |
| import os |
| from expandlibs import ( |
| ExpandArgs, |
| relativize, |
| isDynamicLib, |
| isObject, |
| ) |
| import expandlibs_config as conf |
| from optparse import OptionParser |
| import subprocess |
| import tempfile |
| import shutil |
| import subprocess |
| import re |
| from mozbuild.makeutil import Makefile |
| |
| # The are the insert points for a GNU ld linker script, assuming a more |
| # or less "standard" default linker script. This is not a dict because |
| # order is important. |
| SECTION_INSERT_BEFORE = [ |
| ('.text', '.fini'), |
| ('.rodata', '.rodata1'), |
| ('.data.rel.ro', '.dynamic'), |
| ('.data', '.data1'), |
| ] |
| |
| class ExpandArgsMore(ExpandArgs): |
| ''' Meant to be used as 'with ExpandArgsMore(args) as ...: ''' |
| def __enter__(self): |
| self.tmp = [] |
| return self |
| |
| def __exit__(self, type, value, tb): |
| '''Automatically remove temporary files''' |
| for tmp in self.tmp: |
| if os.path.isdir(tmp): |
| shutil.rmtree(tmp, True) |
| else: |
| os.remove(tmp) |
| |
| def extract(self): |
| self[0:] = self._extract(self) |
| |
| def _extract(self, args): |
| '''When a static library name is found, either extract its contents |
| in a temporary directory or use the information found in the |
| corresponding lib descriptor. |
| ''' |
| ar_extract = conf.AR_EXTRACT.split() |
| newlist = [] |
| |
| def lookup(base, f): |
| for root, dirs, files in os.walk(base): |
| if f in files: |
| return os.path.join(root, f) |
| |
| for arg in args: |
| if os.path.splitext(arg)[1] == conf.LIB_SUFFIX: |
| if os.path.exists(arg + conf.LIBS_DESC_SUFFIX): |
| newlist += self._extract(self._expand_desc(arg)) |
| continue |
| elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'): |
| tmp = tempfile.mkdtemp(dir=os.curdir) |
| self.tmp.append(tmp) |
| if conf.AR == 'lib': |
| out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg]) |
| files = out.splitlines() |
| # If lib -list returns a list full of dlls, it's an |
| # import lib. |
| if all(isDynamicLib(f) for f in files): |
| newlist += [arg] |
| continue |
| for f in files: |
| subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp) |
| else: |
| subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp) |
| objs = [] |
| basedir = os.path.dirname(arg) |
| for root, dirs, files in os.walk(tmp): |
| for f in files: |
| if isObject(f): |
| # If the file extracted from the library also |
| # exists in the directory containing the |
| # library, or one of its subdirectories, use |
| # that instead. |
| maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f) |
| if maybe_obj: |
| objs.append(relativize(maybe_obj)) |
| else: |
| objs.append(relativize(os.path.join(root, f))) |
| newlist += sorted(objs) |
| continue |
| newlist += [arg] |
| return newlist |
| |
| def makelist(self): |
| '''Replaces object file names with a temporary list file, using a |
| list format depending on the EXPAND_LIBS_LIST_STYLE variable |
| ''' |
| objs = [o for o in self if isObject(o)] |
| if not len(objs): return |
| fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir) |
| if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript": |
| content = ['INPUT("%s")\n' % obj for obj in objs] |
| ref = tmp |
| elif conf.EXPAND_LIBS_LIST_STYLE == "filelist": |
| content = ["%s\n" % obj for obj in objs] |
| ref = "-Wl,-filelist," + tmp |
| elif conf.EXPAND_LIBS_LIST_STYLE == "list": |
| content = ["%s\n" % obj for obj in objs] |
| ref = "@" + tmp |
| else: |
| os.close(fd) |
| os.remove(tmp) |
| return |
| self.tmp.append(tmp) |
| f = os.fdopen(fd, "w") |
| f.writelines(content) |
| f.close() |
| idx = self.index(objs[0]) |
| newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs] |
| self[0:] = newlist |
| |
| def _getFoldedSections(self): |
| '''Returns a dict about folded sections. |
| When section A and B are folded into section C, the dict contains: |
| { 'A': 'C', |
| 'B': 'C', |
| 'C': ['A', 'B'] }''' |
| if not conf.LD_PRINT_ICF_SECTIONS: |
| return {} |
| |
| proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE) |
| (stdout, stderr) = proc.communicate() |
| result = {} |
| # gold's --print-icf-sections output looks like the following: |
| # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o' |
| # In terms of words, chances are this will change in the future, |
| # especially considering "into" is misplaced. Splitting on quotes |
| # seems safer. |
| for l in stderr.split('\n'): |
| quoted = l.split("'") |
| if len(quoted) > 5 and quoted[1] != quoted[5]: |
| result[quoted[1]] = [quoted[5]] |
| if quoted[5] in result: |
| result[quoted[5]].append(quoted[1]) |
| else: |
| result[quoted[5]] = [quoted[1]] |
| return result |
| |
| def _getOrderedSections(self, ordered_symbols): |
| '''Given an ordered list of symbols, returns the corresponding list |
| of sections following the order.''' |
| if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: |
| raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) |
| finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX]) |
| folded = self._getFoldedSections() |
| sections = set() |
| ordered_sections = [] |
| for symbol in ordered_symbols: |
| symbol_sections = finder.getSections(symbol) |
| all_symbol_sections = [] |
| for section in symbol_sections: |
| if section in folded: |
| if isinstance(folded[section], str): |
| section = folded[section] |
| all_symbol_sections.append(section) |
| all_symbol_sections.extend(folded[section]) |
| else: |
| all_symbol_sections.append(section) |
| for section in all_symbol_sections: |
| if not section in sections: |
| ordered_sections.append(section) |
| sections.add(section) |
| return ordered_sections |
| |
| def orderSymbols(self, order): |
| '''Given a file containing a list of symbols, adds the appropriate |
| argument to make the linker put the symbols in that order.''' |
| with open(order) as file: |
| sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()]) |
| split_sections = {} |
| linked_sections = [s[0] for s in SECTION_INSERT_BEFORE] |
| for s in sections: |
| for linked_section in linked_sections: |
| if s.startswith(linked_section): |
| if linked_section in split_sections: |
| split_sections[linked_section].append(s) |
| else: |
| split_sections[linked_section] = [s] |
| break |
| content = [] |
| # Order is important |
| linked_sections = [s for s in linked_sections if s in split_sections] |
| |
| if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file': |
| option = '-Wl,--section-ordering-file,%s' |
| content = sections |
| for linked_section in linked_sections: |
| content.extend(split_sections[linked_section]) |
| content.append('%s.*' % linked_section) |
| content.append(linked_section) |
| |
| elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript': |
| option = '-Wl,-T,%s' |
| section_insert_before = dict(SECTION_INSERT_BEFORE) |
| for linked_section in linked_sections: |
| content.append('SECTIONS {') |
| content.append(' %s : {' % linked_section) |
| content.extend(' *(%s)' % s for s in split_sections[linked_section]) |
| content.append(' }') |
| content.append('}') |
| content.append('INSERT BEFORE %s' % section_insert_before[linked_section]) |
| else: |
| raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) |
| |
| fd, tmp = tempfile.mkstemp(dir=os.curdir) |
| f = os.fdopen(fd, "w") |
| f.write('\n'.join(content)+'\n') |
| f.close() |
| self.tmp.append(tmp) |
| self.append(option % tmp) |
| |
| class SectionFinder(object): |
| '''Instances of this class allow to map symbol names to sections in |
| object files.''' |
| |
| def __init__(self, objs): |
| '''Creates an instance, given a list of object files.''' |
| if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: |
| raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) |
| self.mapping = {} |
| for obj in objs: |
| if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX: |
| raise Exception('%s is not an object nor a static library' % obj) |
| for symbol, section in SectionFinder._getSymbols(obj): |
| sym = SectionFinder._normalize(symbol) |
| if sym in self.mapping: |
| if not section in self.mapping[sym]: |
| self.mapping[sym].append(section) |
| else: |
| self.mapping[sym] = [section] |
| |
| def getSections(self, symbol): |
| '''Given a symbol, returns a list of sections containing it or the |
| corresponding thunks. When the given symbol is a thunk, returns the |
| list of sections containing its corresponding normal symbol and the |
| other thunks for that symbol.''' |
| sym = SectionFinder._normalize(symbol) |
| if sym in self.mapping: |
| return self.mapping[sym] |
| return [] |
| |
| @staticmethod |
| def _normalize(symbol): |
| '''For normal symbols, return the given symbol. For thunks, return |
| the corresponding normal symbol.''' |
| if re.match('^_ZThn[0-9]+_', symbol): |
| return re.sub('^_ZThn[0-9]+_', '_Z', symbol) |
| return symbol |
| |
| @staticmethod |
| def _getSymbols(obj): |
| '''Returns a list of (symbol, section) contained in the given object |
| file.''' |
| proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE) |
| (stdout, stderr) = proc.communicate() |
| syms = [] |
| for line in stdout.splitlines(): |
| # Each line has the following format: |
| # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol> |
| tmp = line.split(' ',1) |
| # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"] |
| # We only need to consider cases where "<section>\t<length> <symbol>" is present, |
| # and where the [FfO] flag is either F (function) or O (object). |
| if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']: |
| tmp = tmp[1][8:].split() |
| # That gives us ["<section>","<length>", "<symbol>"] |
| syms.append((tmp[-1], tmp[0])) |
| return syms |
| |
| def print_command(out, args): |
| print >>out, "Executing: " + " ".join(args) |
| for tmp in [f for f in args.tmp if os.path.isfile(f)]: |
| print >>out, tmp + ":" |
| with open(tmp) as file: |
| print >>out, "".join([" " + l for l in file.readlines()]) |
| out.flush() |
| |
| def main(args, proc_callback=None): |
| parser = OptionParser() |
| parser.add_option("--extract", action="store_true", dest="extract", |
| help="when a library has no descriptor file, extract it first, when possible") |
| parser.add_option("--uselist", action="store_true", dest="uselist", |
| help="use a list file for objects when executing a command") |
| parser.add_option("--verbose", action="store_true", dest="verbose", |
| help="display executed command and temporary files content") |
| parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE", |
| help="use the given list of symbols to order symbols in the resulting binary when using with a linker") |
| |
| (options, args) = parser.parse_args(args) |
| |
| with ExpandArgsMore(args) as args: |
| if options.extract: |
| args.extract() |
| if options.symbol_order: |
| args.orderSymbols(options.symbol_order) |
| if options.uselist: |
| args.makelist() |
| |
| if options.verbose: |
| print_command(sys.stderr, args) |
| try: |
| proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) |
| if proc_callback: |
| proc_callback(proc) |
| except Exception, e: |
| print >>sys.stderr, 'error: Launching', args, ':', e |
| raise e |
| (stdout, stderr) = proc.communicate() |
| if proc.returncode and not options.verbose: |
| print_command(sys.stderr, args) |
| sys.stderr.write(stdout) |
| sys.stderr.flush() |
| if proc.returncode: |
| return proc.returncode |
| return 0 |
| |
| if __name__ == '__main__': |
| exit(main(sys.argv[1:])) |
