build/android/pylib/local/machine/local_machine_junit_test_run.py

# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import collections
import json
import logging
import multiprocessing
import os
import select
import subprocess
import sys
import zipfile

from six.moves import range  # pylint: disable=redefined-builtin
from pylib import constants
from pylib.base import base_test_result
from pylib.base import test_run
from pylib.constants import host_paths
from pylib.results import json_results
from py_utils import tempfile_ext


# These Test classes are used for running tests and are excluded in the test
# runner. See:
# https://android.googlesource.com/platform/frameworks/testing/+/android-support-test/runner/src/main/java/android/support/test/internal/runner/TestRequestBuilder.java
# base/test/android/javatests/src/org/chromium/base/test/BaseChromiumAndroidJUnitRunner.java # pylint: disable=line-too-long
_EXCLUDED_CLASSES_PREFIXES = ('android', 'junit', 'org/bouncycastle/util',
                              'org/hamcrest', 'org/junit', 'org/mockito')

# Suites we shouldn't shard, usually because they don't contain enough test
# cases.
_EXCLUDED_SUITES = {
    'password_check_junit_tests',
    'touch_to_fill_junit_tests',
}


# It can actually take longer to run if you shard too much, especially on
# smaller suites. Locally media_base_junit_tests takes 4.3 sec with 1 shard,
# and 6 sec with 2 or more shards.
_MIN_CLASSES_PER_SHARD = 8


class LocalMachineJunitTestRun(test_run.TestRun):
  def __init__(self, env, test_instance):
    super(LocalMachineJunitTestRun, self).__init__(env, test_instance)

  #override
  def TestPackage(self):
    return self._test_instance.suite

  #override
  def SetUp(self):
    pass

  def _CreateJarArgsList(self, json_result_file_paths, group_test_list, shards):
    # Creates a list of jar_args. The important thing is each jar_args list
    # has a different json_results file for writing test results to and that
    # each list of jar_args has its own test to run as specified in the
    # -gtest-filter.
    jar_args_list = [['-json-results-file', result_file]
                     for result_file in json_result_file_paths]
    for index, jar_arg in enumerate(jar_args_list):
      if shards > 1:
        jar_arg.extend(['-gtest-filter', ':'.join(group_test_list[index])])
      elif self._test_instance.test_filter:
        jar_arg.extend(['-gtest-filter', self._test_instance.test_filter])

      if self._test_instance.package_filter:
        jar_arg.extend(['-package-filter', self._test_instance.package_filter])
      if self._test_instance.runner_filter:
        jar_arg.extend(['-runner-filter', self._test_instance.runner_filter])

    return jar_args_list

  def _CreateJvmArgsList(self):
    # Creates a list of jvm_args (robolectric, code coverage, etc...)
    jvm_args = [
        '-Drobolectric.dependency.dir=%s' %
        self._test_instance.robolectric_runtime_deps_dir,
        '-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT,
        '-Drobolectric.resourcesMode=binary',
    ]
    if logging.getLogger().isEnabledFor(logging.INFO):
      jvm_args += ['-Drobolectric.logging=stdout']
    if self._test_instance.debug_socket:
      jvm_args += [
          '-agentlib:jdwp=transport=dt_socket'
          ',server=y,suspend=y,address=%s' % self._test_instance.debug_socket
      ]

    if self._test_instance.coverage_dir:
      if not os.path.exists(self._test_instance.coverage_dir):
        os.makedirs(self._test_instance.coverage_dir)
      elif not os.path.isdir(self._test_instance.coverage_dir):
        raise Exception('--coverage-dir takes a directory, not file path.')
      if self._test_instance.coverage_on_the_fly:
        jacoco_coverage_file = os.path.join(
            self._test_instance.coverage_dir,
            '%s.exec' % self._test_instance.suite)
        jacoco_agent_path = os.path.join(host_paths.DIR_SOURCE_ROOT,
                                         'third_party', 'jacoco', 'lib',
                                         'jacocoagent.jar')

        # inclnolocationclasses is false to prevent no class def found error.
        jacoco_args = '-javaagent:{}=destfile={},inclnolocationclasses=false'
        jvm_args.append(
            jacoco_args.format(jacoco_agent_path, jacoco_coverage_file))
      else:
        jvm_args.append('-Djacoco-agent.destfile=%s' %
                        os.path.join(self._test_instance.coverage_dir,
                                     '%s.exec' % self._test_instance.suite))

    return jvm_args

  #override
  def RunTests(self, results):
    wrapper_path = os.path.join(constants.GetOutDirectory(), 'bin', 'helper',
                                self._test_instance.suite)

    # This avoids searching through the classparth jars for tests classes,
    # which takes about 1-2 seconds.
    # Do not shard when a test filter is present since we do not know at this
    # point which tests will be filtered out.
    if (self._test_instance.shards == 1 or self._test_instance.test_filter
        or self._test_instance.suite in _EXCLUDED_SUITES):
      test_classes = []
      shards = 1
    else:
      test_classes = _GetTestClasses(wrapper_path)
      shards = ChooseNumOfShards(test_classes, self._test_instance.shards)

    logging.info('Running tests on %d shard(s).', shards)
    group_test_list = GroupTestsForShard(shards, test_classes)

    with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
      cmd_list = [[wrapper_path] for _ in range(shards)]
      json_result_file_paths = [
          os.path.join(temp_dir, 'results%d.json' % i) for i in range(shards)
      ]
      jar_args_list = self._CreateJarArgsList(json_result_file_paths,
                                              group_test_list, shards)
      for i in range(shards):
        cmd_list[i].extend(['--jar-args', '"%s"' % ' '.join(jar_args_list[i])])

      jvm_args = self._CreateJvmArgsList()
      if jvm_args:
        for cmd in cmd_list:
          cmd.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)])

      AddPropertiesJar(cmd_list, temp_dir, self._test_instance.resource_apk)

      procs = [
          subprocess.Popen(cmd,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT) for cmd in cmd_list
      ]
      PrintProcessesStdout(procs)

      results_list = []
      try:
        for json_file_path in json_result_file_paths:
          with open(json_file_path, 'r') as f:
            results_list += json_results.ParseResultsFromJson(
                json.loads(f.read()))
      except IOError:
        # In the case of a failure in the JUnit or Robolectric test runner
        # the output json file may never be written.
        results_list = [
          base_test_result.BaseTestResult(
              'Test Runner Failure', base_test_result.ResultType.UNKNOWN)
        ]

      test_run_results = base_test_result.TestRunResults()
      test_run_results.AddResults(results_list)
      results.append(test_run_results)

  #override
  def TearDown(self):
    pass


def AddPropertiesJar(cmd_list, temp_dir, resource_apk):
  # Create properties file for Robolectric test runners so they can find the
  # binary resources.
  properties_jar_path = os.path.join(temp_dir, 'properties.jar')
  with zipfile.ZipFile(properties_jar_path, 'w') as z:
    z.writestr('com/android/tools/test_config.properties',
               'android_resource_apk=%s' % resource_apk)

  for cmd in cmd_list:
    cmd.extend(['--classpath', properties_jar_path])


def ChooseNumOfShards(test_classes, shards):
  # Don't override requests to not shard.
  if shards == 1:
    return 1

  # Sharding doesn't reduce runtime on just a few tests.
  if shards > (len(test_classes) // _MIN_CLASSES_PER_SHARD) or shards < 1:
    shards = max(1, (len(test_classes) // _MIN_CLASSES_PER_SHARD))

  # Local tests of explicit --shard values show that max speed is achieved
  # at cpu_count() / 2.
  # Using -XX:TieredStopAtLevel=1 is required for this result. The flag reduces
  # CPU time by two-thirds, making sharding more effective.
  shards = max(1, min(shards, multiprocessing.cpu_count() // 2))
  # Can have at minimum one test_class per shard.
  shards = min(len(test_classes), shards)

  return shards


def GroupTestsForShard(num_of_shards, test_classes):
  """Groups tests that will be ran on each shard.

  Args:
    num_of_shards: number of shards to split tests between.
    test_classes: A list of test_class files in the jar.

  Return:
    Returns a dictionary containing a list of test classes.
  """
  test_dict = {i: [] for i in range(num_of_shards)}

  # Round robin test distribiution to reduce chance that a sequential group of
  # classes all have an unusually high number of tests.
  for count, test_cls in enumerate(test_classes):
    test_cls = test_cls.replace('.class', '*')
    test_cls = test_cls.replace('/', '.')
    test_dict[count % num_of_shards].append(test_cls)

  return test_dict


def PrintProcessesStdout(procs):
  """Prints the stdout of all the processes.

  Buffers the stdout of the processes and prints it when finished.

  Args:
    procs: A list of subprocesses.

  Returns: N/A
  """
  streams = [p.stdout for p in procs]
  outputs = collections.defaultdict(list)
  first_fd = streams[0].fileno()

  while streams:
    rstreams, _, _ = select.select(streams, [], [])
    for stream in rstreams:
      line = stream.readline()
      if line:
        # Print out just one output so user can see work being done rather
        # than waiting for it all at the end.
        if stream.fileno() == first_fd:
          sys.stdout.write(line)
        else:
          outputs[stream.fileno()].append(line)
      else:
        streams.remove(stream)  # End of stream.

  for p in procs:
    sys.stdout.write(''.join(outputs[p.stdout.fileno()]))


def _GetTestClasses(file_path):
  test_jar_paths = subprocess.check_output([file_path, '--print-classpath'])
  test_jar_paths = test_jar_paths.split(':')

  test_classes = []
  for test_jar_path in test_jar_paths:
    # Avoid searching through jars that are for the test runner.
    # TODO(crbug.com/1144077): Use robolectric buildconfig file arg.
    if 'third_party/robolectric/' in test_jar_path:
      continue

    test_classes += _GetTestClassesFromJar(test_jar_path)

  logging.info('Found %d test classes in class_path jars.', len(test_classes))
  return test_classes


def _GetTestClassesFromJar(test_jar_path):
  """Returns a list of test classes from a jar.

  Test files end in Test, this is enforced:
  //tools/android/errorprone_plugin/src/org/chromium/tools/errorprone
  /plugin/TestClassNameCheck.java

  Args:
    test_jar_path: Path to the jar.

  Return:
    Returns a list of test classes that were in the jar.
  """
  class_list = []
  with zipfile.ZipFile(test_jar_path, 'r') as zip_f:
    for test_class in zip_f.namelist():
      if test_class.startswith(_EXCLUDED_CLASSES_PREFIXES):
        continue
      if test_class.endswith('Test.class') and '$' not in test_class:
        class_list.append(test_class)

  return class_list