blob: 579501c7ded30e0073ee6ac512ed733f5c5b0c48 [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import logging
import os
import pathlib
import re
import shutil
import sys
import zipfile
import dex
from util import build_utils
from util import diff_utils
import action_helpers # build_utils adds //build to sys.path.
import zip_helpers
_BLOCKLISTED_EXPECTATION_PATHS = [
# A separate expectation file is created for these files.
'clank/third_party/google3/pg_confs/',
]
_DUMP_DIR_NAME = 'r8inputs_dir'
def _ParseOptions():
args = build_utils.ExpandFileArgs(sys.argv[1:])
parser = argparse.ArgumentParser()
action_helpers.add_depfile_arg(parser)
parser.add_argument('--r8-path',
required=True,
help='Path to the R8.jar to use.')
parser.add_argument('--input-paths',
action='append',
required=True,
help='GN-list of .jar files to optimize.')
parser.add_argument('--output-path', help='Path to the generated .jar file.')
parser.add_argument(
'--proguard-configs',
action='append',
required=True,
help='GN-list of configuration files.')
parser.add_argument(
'--apply-mapping', help='Path to ProGuard mapping to apply.')
parser.add_argument(
'--mapping-output',
required=True,
help='Path for ProGuard to output mapping file to.')
parser.add_argument(
'--extra-mapping-output-paths',
help='GN-list of additional paths to copy output mapping file to.')
parser.add_argument(
'--classpath',
action='append',
help='GN-list of .jar files to include as libraries.')
parser.add_argument('--main-dex-rules-path',
action='append',
help='Path to main dex rules for multidex.')
parser.add_argument(
'--min-api', help='Minimum Android API level compatibility.')
parser.add_argument('--enable-obfuscation',
action='store_true',
help='Minify symbol names')
parser.add_argument(
'--verbose', '-v', action='store_true', help='Print all ProGuard output')
parser.add_argument(
'--repackage-classes', help='Package all optimized classes are put in.')
parser.add_argument(
'--disable-checks',
action='store_true',
help='Disable -checkdiscard directives and missing symbols check')
parser.add_argument('--source-file', help='Value for source file attribute.')
parser.add_argument('--package-name',
help='Goes into a comment in the mapping file.')
parser.add_argument(
'--force-enable-assertions',
action='store_true',
help='Forcefully enable javac generated assertion code.')
parser.add_argument('--assertion-handler',
help='The class name of the assertion handler class.')
parser.add_argument(
'--feature-jars',
action='append',
help='GN list of path to jars which comprise the corresponding feature.')
parser.add_argument(
'--dex-dest',
action='append',
dest='dex_dests',
help='Destination for dex file of the corresponding feature.')
parser.add_argument(
'--feature-name',
action='append',
dest='feature_names',
help='The name of the feature module.')
parser.add_argument(
'--uses-split',
action='append',
help='List of name pairs separated by : mapping a feature module to a '
'dependent feature module.')
parser.add_argument(
'--keep-rules-targets-regex',
metavar='KEEP_RULES_REGEX',
help='If passed outputs keep rules for references from all other inputs '
'to the subset of inputs that satisfy the KEEP_RULES_REGEX.')
parser.add_argument(
'--keep-rules-output-path',
help='Output path to the keep rules for references to the '
'--keep-rules-targets-regex inputs from the rest of the inputs.')
parser.add_argument('--warnings-as-errors',
action='store_true',
help='Treat all warnings as errors.')
parser.add_argument('--show-desugar-default-interface-warnings',
action='store_true',
help='Enable desugaring warnings.')
parser.add_argument('--dump-inputs',
action='store_true',
help='Use when filing R8 bugs to capture inputs.'
' Stores inputs to r8inputs.zip')
parser.add_argument(
'--dump-unknown-refs',
action='store_true',
help='Log all reasons why API modelling cannot determine API level')
parser.add_argument(
'--stamp',
help='File to touch upon success. Mutually exclusive with --output-path')
parser.add_argument('--desugared-library-keep-rule-output',
help='Path to desugared library keep rule output file.')
diff_utils.AddCommandLineFlags(parser)
options = parser.parse_args(args)
if options.feature_names:
if options.output_path:
parser.error('Feature splits cannot specify an output in GN.')
if not options.actual_file and not options.stamp:
parser.error('Feature splits require a stamp file as output.')
elif not options.output_path:
parser.error('Output path required when feature splits aren\'t used')
if bool(options.keep_rules_targets_regex) != bool(
options.keep_rules_output_path):
parser.error('You must path both --keep-rules-targets-regex and '
'--keep-rules-output-path')
if options.force_enable_assertions and options.assertion_handler:
parser.error('Cannot use both --force-enable-assertions and '
'--assertion-handler')
options.classpath = action_helpers.parse_gn_list(options.classpath)
options.proguard_configs = action_helpers.parse_gn_list(
options.proguard_configs)
options.input_paths = action_helpers.parse_gn_list(options.input_paths)
options.extra_mapping_output_paths = action_helpers.parse_gn_list(
options.extra_mapping_output_paths)
if options.feature_names:
if 'base' not in options.feature_names:
parser.error('"base" feature required when feature arguments are used.')
if len(options.feature_names) != len(options.feature_jars) or len(
options.feature_names) != len(options.dex_dests):
parser.error('Invalid feature argument lengths.')
options.feature_jars = [
action_helpers.parse_gn_list(x) for x in options.feature_jars
]
split_map = {}
if options.uses_split:
for split_pair in options.uses_split:
child, parent = split_pair.split(':')
for name in (child, parent):
if name not in options.feature_names:
parser.error('"%s" referenced in --uses-split not present.' % name)
split_map[child] = parent
options.uses_split = split_map
return options
class _SplitContext:
def __init__(self, name, output_path, input_jars, work_dir, parent_name=None):
self.name = name
self.parent_name = parent_name
self.input_jars = set(input_jars)
self.final_output_path = output_path
self.staging_dir = os.path.join(work_dir, name)
os.mkdir(self.staging_dir)
def CreateOutput(self):
found_files = build_utils.FindInDirectory(self.staging_dir)
if not found_files:
raise Exception('Missing dex outputs in {}'.format(self.staging_dir))
if self.final_output_path.endswith('.dex'):
if len(found_files) != 1:
raise Exception('Expected exactly 1 dex file output, found: {}'.format(
'\t'.join(found_files)))
shutil.move(found_files[0], self.final_output_path)
return
# Add to .jar using Python rather than having R8 output to a .zip directly
# in order to disable compression of the .jar, saving ~500ms.
tmp_jar_output = self.staging_dir + '.jar'
zip_helpers.add_files_to_zip(found_files,
tmp_jar_output,
base_dir=self.staging_dir)
shutil.move(tmp_jar_output, self.final_output_path)
def _OptimizeWithR8(options,
config_paths,
libraries,
dynamic_config_data,
print_stdout=False):
with build_utils.TempDir() as tmp_dir:
if dynamic_config_data:
dynamic_config_path = os.path.join(tmp_dir, 'dynamic_config.flags')
with open(dynamic_config_path, 'w') as f:
f.write(dynamic_config_data)
config_paths = config_paths + [dynamic_config_path]
tmp_mapping_path = os.path.join(tmp_dir, 'mapping.txt')
# If there is no output (no classes are kept), this prevents this script
# from failing.
build_utils.Touch(tmp_mapping_path)
tmp_output = os.path.join(tmp_dir, 'r8out')
os.mkdir(tmp_output)
split_contexts_by_name = {}
if options.feature_names:
for name, dest_dex, input_jars in zip(options.feature_names,
options.dex_dests,
options.feature_jars):
parent_name = options.uses_split.get(name)
if parent_name is None and name != 'base':
parent_name = 'base'
split_context = _SplitContext(name,
dest_dex,
input_jars,
tmp_output,
parent_name=parent_name)
split_contexts_by_name[name] = split_context
else:
# Base context will get populated via "extra_jars" below.
split_contexts_by_name['base'] = _SplitContext('base',
options.output_path, [],
tmp_output)
base_context = split_contexts_by_name['base']
# R8 OOMs with the default xmx=1G.
cmd = build_utils.JavaCmd(xmx='2G') + [
# Allows -whyareyounotinlining, which we don't have by default, but
# which is useful for one-off queries.
'-Dcom.android.tools.r8.experimental.enablewhyareyounotinlining=1',
# Restricts horizontal class merging to apply only to classes that
# share a .java file (nested classes). https://crbug.com/1363709
'-Dcom.android.tools.r8.enableSameFilePolicy=1',
]
if options.dump_inputs:
cmd += [f'-Dcom.android.tools.r8.dumpinputtodirectory={_DUMP_DIR_NAME}']
if options.dump_unknown_refs:
cmd += ['-Dcom.android.tools.r8.reportUnknownApiReferences=1']
cmd += [
'-cp',
options.r8_path,
'com.android.tools.r8.R8',
'--no-data-resources',
'--map-id-template',
f'{options.source_file} ({options.package_name})',
'--source-file-template',
options.source_file,
'--output',
base_context.staging_dir,
'--pg-map-output',
tmp_mapping_path,
]
if options.disable_checks:
cmd += ['--map-diagnostics:CheckDiscardDiagnostic', 'error', 'none']
cmd += ['--map-diagnostics', 'info', 'warning']
# An "error" level diagnostic causes r8 to return an error exit code. Doing
# this allows our filter to decide what should/shouldn't break our build.
cmd += ['--map-diagnostics', 'error', 'warning']
if options.min_api:
cmd += ['--min-api', options.min_api]
if options.assertion_handler:
cmd += ['--force-assertions-handler:' + options.assertion_handler]
elif options.force_enable_assertions:
cmd += ['--force-enable-assertions']
for lib in libraries:
cmd += ['--lib', lib]
for config_file in config_paths:
cmd += ['--pg-conf', config_file]
if options.main_dex_rules_path:
for main_dex_rule in options.main_dex_rules_path:
cmd += ['--main-dex-rules', main_dex_rule]
# Add any extra inputs to the base context (e.g. desugar runtime).
extra_jars = set(options.input_paths)
for split_context in split_contexts_by_name.values():
extra_jars -= split_context.input_jars
base_context.input_jars.update(extra_jars)
for split_context in split_contexts_by_name.values():
if split_context is base_context:
continue
for in_jar in sorted(split_context.input_jars):
cmd += ['--feature', in_jar, split_context.staging_dir]
cmd += sorted(base_context.input_jars)
try:
stderr_filter = dex.CreateStderrFilter(
options.show_desugar_default_interface_warnings)
logging.debug('Running R8')
build_utils.CheckOutput(cmd,
print_stdout=print_stdout,
stderr_filter=stderr_filter,
fail_on_output=options.warnings_as_errors)
except build_utils.CalledProcessError as e:
# Do not output command line because it is massive and makes the actual
# error message hard to find.
sys.stderr.write(e.output)
sys.exit(1)
logging.debug('Collecting ouputs')
base_context.CreateOutput()
for split_context in split_contexts_by_name.values():
if split_context is not base_context:
split_context.CreateOutput()
shutil.move(tmp_mapping_path, options.mapping_output)
return split_contexts_by_name
def _OutputKeepRules(r8_path, input_paths, classpath, targets_re_string,
keep_rules_output):
cmd = build_utils.JavaCmd() + [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning',
'--keep-rules', '--output', keep_rules_output
]
targets_re = re.compile(targets_re_string)
for path in input_paths:
if targets_re.search(path):
cmd += ['--target', path]
else:
cmd += ['--source', path]
for path in classpath:
cmd += ['--lib', path]
build_utils.CheckOutput(cmd, print_stderr=False, fail_on_output=False)
def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors,
dump_inputs, error_title):
cmd = build_utils.JavaCmd()
if dump_inputs:
cmd += [f'-Dcom.android.tools.r8.dumpinputtodirectory={_DUMP_DIR_NAME}']
cmd += [
'-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences',
'--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning',
'--check'
]
for path in classpath:
cmd += ['--lib', path]
for path in dex_files:
cmd += ['--source', path]
failed_holder = [False]
def stderr_filter(stderr):
ignored_lines = [
# Summary contains warning count, which our filtering makes wrong.
'Warning: Tracereferences found',
# TODO(agrieve): Create interface jars for these missing classes rather
# than allowlisting here.
'dalvik.system',
'libcore.io',
'sun.misc.Unsafe',
# Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper
'android.text.StaticLayout.<init>',
# TODO(crbug/1426964): Remove once chrome builds with Android U SDK.
'android.adservices.measurement',
# Explicictly guarded by try (NoClassDefFoundError) in Flogger's
# PlatformProvider.
'com.google.common.flogger.backend.google.GooglePlatform',
'com.google.common.flogger.backend.system.DefaultPlatform',
# TODO(agrieve): Exclude these only when use_jacoco_coverage=true.
'java.lang.instrument.ClassFileTransformer',
'java.lang.instrument.IllegalClassFormatException',
'java.lang.instrument.Instrumentation',
'java.lang.management.ManagementFactory',
'javax.management.MBeanServer',
'javax.management.ObjectInstance',
'javax.management.ObjectName',
'javax.management.StandardMBean',
# Explicitly guarded by try (NoClassDefFoundError) in Firebase's
# KotlinDetector: com.google.firebase.platforminfo.KotlinDetector.
'kotlin.KotlinVersion',
# TODO(agrieve): Remove once we move to Android U SDK.
'android.window.BackEvent',
'android.window.OnBackAnimationCallback',
]
had_unfiltered_items = ' ' in stderr
stderr = build_utils.FilterLines(
stderr, '|'.join(re.escape(x) for x in ignored_lines))
if stderr:
if 'Missing' in stderr:
failed_holder[0] = True
stderr = 'TraceReferences failed: ' + error_title + """
Tip: Build with:
is_java_debug=false
treat_warnings_as_errors=false
enable_proguard_obfuscation=false
and then use dexdump to see which class(s) reference them.
E.g.:
third_party/android_sdk/public/build-tools/*/dexdump -d \
out/Release/apks/YourApk.apk > dex.txt
""" + stderr
if 'FragmentActivity' in stderr:
stderr += """
You may need to update build configs to run FragmentActivityReplacer for
additional targets. See
https://chromium.googlesource.com/chromium/src.git/+/main/docs/ui/android/bytecode_rewriting.md.
"""
elif had_unfiltered_items:
# Left only with empty headings. All indented items filtered out.
stderr = ''
return stderr
try:
build_utils.CheckOutput(cmd,
print_stdout=True,
stderr_filter=stderr_filter,
fail_on_output=warnings_as_errors)
except build_utils.CalledProcessError as e:
# Do not output command line because it is massive and makes the actual
# error message hard to find.
sys.stderr.write(e.output)
sys.exit(1)
return failed_holder[0]
def _CombineConfigs(configs,
dynamic_config_data,
embedded_configs,
exclude_generated=False):
# Sort in this way so //clank versions of the same libraries will sort
# to the same spot in the file.
def sort_key(path):
return tuple(reversed(path.split(os.path.sep)))
def format_config_contents(path, contents):
formatted_contents = []
if not contents.strip():
return []
# Fix up line endings (third_party configs can have windows endings).
contents = contents.replace('\r', '')
# Remove numbers from generated rule comments to make file more
# diff'able.
contents = re.sub(r' #generated:\d+', '', contents)
formatted_contents.append('# File: ' + path)
formatted_contents.append(contents)
formatted_contents.append('')
return formatted_contents
ret = []
for config in sorted(configs, key=sort_key):
if exclude_generated and config.endswith('.resources.proguard.txt'):
continue
# Exclude some confs from expectations.
if any(entry in config for entry in _BLOCKLISTED_EXPECTATION_PATHS):
continue
with open(config) as config_file:
contents = config_file.read().rstrip()
ret.extend(format_config_contents(config, contents))
for path, contents in sorted(embedded_configs.items()):
ret.extend(format_config_contents(path, contents))
if dynamic_config_data:
ret.append('# File: //build/android/gyp/proguard.py (generated rules)')
ret.append(dynamic_config_data)
ret.append('')
return '\n'.join(ret)
def _CreateDynamicConfig(options):
ret = []
if options.enable_obfuscation:
ret.append("-repackageclasses ''")
else:
ret.append("-dontobfuscate")
if options.apply_mapping:
ret.append("-applymapping '%s'" % options.apply_mapping)
return '\n'.join(ret)
def _ExtractEmbeddedConfigs(jar_path, embedded_configs):
with zipfile.ZipFile(jar_path) as z:
proguard_names = []
r8_names = []
for info in z.infolist():
if info.is_dir():
continue
if info.filename.startswith('META-INF/proguard/'):
proguard_names.append(info.filename)
elif info.filename.startswith('META-INF/com.android.tools/r8/'):
r8_names.append(info.filename)
elif info.filename.startswith('META-INF/com.android.tools/r8-from'):
# Assume our version of R8 is always latest.
if '-upto-' not in info.filename:
r8_names.append(info.filename)
# Give preference to r8-from-*, then r8/, then proguard/.
active = r8_names or proguard_names
for filename in active:
config_path = '{}:{}'.format(jar_path, filename)
embedded_configs[config_path] = z.read(filename).decode('utf-8').rstrip()
def _ContainsDebuggingConfig(config_str):
debugging_configs = ('-whyareyoukeeping', '-whyareyounotinlining')
return any(config in config_str for config in debugging_configs)
def _MaybeWriteStampAndDepFile(options, inputs):
output = options.output_path
if options.stamp:
build_utils.Touch(options.stamp)
output = options.stamp
if options.depfile:
action_helpers.write_depfile(options.depfile, output, inputs=inputs)
def _IterParentContexts(context_name, split_contexts_by_name):
while context_name:
context = split_contexts_by_name[context_name]
yield context
context_name = context.parent_name
def _DoTraceReferencesChecks(options, split_contexts_by_name):
# Set of all contexts that are a parent to another.
parent_splits_context_names = {
c.parent_name
for c in split_contexts_by_name.values() if c.parent_name
}
context_sets = [
list(_IterParentContexts(n, split_contexts_by_name))
for n in parent_splits_context_names
]
# Visit them in order of: base, base+chrome, base+chrome+thing.
context_sets.sort(key=lambda x: (len(x), x[0].name))
# Ensure there are no missing references when considering all dex files.
error_title = 'DEX contains references to non-existent symbols after R8.'
dex_files = sorted(c.final_output_path
for c in split_contexts_by_name.values())
if _CheckForMissingSymbols(options.r8_path, dex_files, options.classpath,
options.warnings_as_errors, options.dump_inputs,
error_title):
# Failed but didn't raise due to warnings_as_errors=False
return
for context_set in context_sets:
# Ensure there are no references from base -> chrome module, or from
# chrome -> feature modules.
error_title = (f'DEX within module "{context_set[0].name}" contains '
'reference(s) to symbols within child splits')
dex_files = [c.final_output_path for c in context_set]
# Each check currently takes about 3 seconds on a fast dev machine, and we
# run 3 of them (all, base, base+chrome).
# We could run them concurrently, to shave off 5-6 seconds, but would need
# to make sure that the order is maintained.
if _CheckForMissingSymbols(options.r8_path, dex_files, options.classpath,
options.warnings_as_errors, options.dump_inputs,
error_title):
# Failed but didn't raise due to warnings_as_errors=False
return
def _Run(options):
# ProGuard configs that are derived from flags.
logging.debug('Preparing configs')
dynamic_config_data = _CreateDynamicConfig(options)
logging.debug('Looking for embedded configs')
# If a jar is part of input no need to include it as library jar.
libraries = [p for p in options.classpath if p not in options.input_paths]
embedded_configs = {}
for jar_path in options.input_paths + libraries:
_ExtractEmbeddedConfigs(jar_path, embedded_configs)
# ProGuard configs that are derived from flags.
merged_configs = _CombineConfigs(options.proguard_configs,
dynamic_config_data,
embedded_configs,
exclude_generated=True)
print_stdout = _ContainsDebuggingConfig(merged_configs) or options.verbose
depfile_inputs = options.proguard_configs + options.input_paths + libraries
if options.expected_file:
diff_utils.CheckExpectations(merged_configs, options)
if options.only_verify_expectations:
action_helpers.write_depfile(options.depfile,
options.actual_file,
inputs=depfile_inputs)
return
if options.keep_rules_output_path:
_OutputKeepRules(options.r8_path, options.input_paths, options.classpath,
options.keep_rules_targets_regex,
options.keep_rules_output_path)
return
split_contexts_by_name = _OptimizeWithR8(options, options.proguard_configs,
libraries, dynamic_config_data,
print_stdout)
if not options.disable_checks:
logging.debug('Running tracereferences')
_DoTraceReferencesChecks(options, split_contexts_by_name)
for output in options.extra_mapping_output_paths:
shutil.copy(options.mapping_output, output)
if options.apply_mapping:
depfile_inputs.append(options.apply_mapping)
_MaybeWriteStampAndDepFile(options, depfile_inputs)
def main():
build_utils.InitLogging('PROGUARD_DEBUG')
options = _ParseOptions()
if options.dump_inputs:
# Dumping inputs causes output to be emitted, avoid failing due to stdout.
options.warnings_as_errors = False
# Use dumpinputtodirectory instead of dumpinputtofile to avoid failing the
# build and keep running tracereferences.
dump_dir_name = _DUMP_DIR_NAME
dump_dir_path = pathlib.Path(dump_dir_name)
if dump_dir_path.exists():
shutil.rmtree(dump_dir_path)
# The directory needs to exist before r8 adds the zip files in it.
dump_dir_path.mkdir()
# This ensure that the final outputs are zipped and easily uploaded to a bug.
try:
_Run(options)
finally:
if options.dump_inputs:
zip_helpers.zip_directory('r8inputs.zip', _DUMP_DIR_NAME)
if __name__ == '__main__':
main()